rsshub 1.0.0-master.fe34161 → 1.0.0-master.fe416a0

package/lib/routes/txrjy/fornumtopic.ts

@@ -1,6 +1,4 @@
 import { Route } from '@/types';
-import { getCurrentPath } from '@/utils/helpers';
-const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
 import got from '@/utils/got';
@@ -45,15 +43,15 @@ async function handler(ctx) {
     const title = $('div.z > a').last().text();
     const list = $('tbody > tr')
         .slice(0, 25)
-        .map((_, item) => ({
+        .toArray()
+        .map((item) => ({
             title: $(item).find('td.title2').text(),
             link: new URL($(item).find('td.title2 > a').attr('href'), rootUrl).href,
             author: $(item).find('td.author').text(),
             pubDate: timezone(parseDate($(item).find('td.dateline').text(), 'YYYY-M-D HH:mm'), +8),
             category: $(item).find('td.forum').text(),
         }))
-        .filter((_, item) => item.title)
-        .get();
+        .filter((item) => item.title);
 
     const items = await Promise.all(
         list.map((item) =>
@@ -64,7 +62,8 @@ async function handler(ctx) {
                 const content = load(iconv.decode(detailResponse.data, 'gbk'));
 
                 item.description = content('div.c_table')
-                    .map((_, item) =>
+                    .toArray()
+                    .map((item) =>
                         art(path.join(__dirname, 'templates/fornumtopic.art'), {
                             content: content(item)
                                 .find('td.t_f')
@@ -72,17 +71,16 @@ async function handler(ctx) {
                                 .remove()
                                 .end()
                                 .html()
-                                ?.replace(/(<img.*?) src=".*?"(.*?>)/g, '$1$2')
+                                ?.replaceAll(/(<img.*?) src=".*?"(.*?>)/g, '$1$2')
                                 .replaceAll(/(<img.*?)zoomfile(.*?>)/g, '$1src$2'),
                             pattl: content(item)
                                 .find('div.pattl')
                                 .html()
-                                ?.replace(/(<img.*?) src=".*?"(.*?>)/g, '$1$2')
+                                ?.replaceAll(/(<img.*?) src=".*?"(.*?>)/g, '$1$2')
                                 .replaceAll(/(<img.*?)zoomfile(.*?>)/g, '$1src$2'),
                             author: content(item).find('a.xw1').text().trim(),
                         })
                     )
-                    .get()
                     .join('\n');
 
                 return item;

package/lib/routes/typst/universe.ts

@@ -77,7 +77,7 @@ export const route: Route = {
                 displayErrors: true,
             });
             const md = markdownit('commonmark');
-            const items = context.an.exports.sort((a, b) => a.updatedAt - b.updatedAt);
+            const items = context.an.exports.toSorted((a, b) => a.updatedAt - b.updatedAt);
             const groups = new Map(items.map((it) => [it.name, it]));
             const pkgs = [...groups.values()].map((item) => {
                 const $ = load(md.render(item.readme));

package/lib/routes/uber/blog.ts

@@ -1,16 +1,17 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 
 const rootURL = 'https://www.uber.com';
-const apiURL = 'https://blogapi.uber.com';
 
 export const route: Route = {
-    path: '/blog/:maxPage?',
+    // `compat` is a never used parameter
+    // just for backward compatibility with the deprecated `:maxPage` parameter
+    path: '/blog/:compat?',
     categories: ['blog'],
     example: '/uber/blog',
-    parameters: { maxPage: 'max number of pages to retrieve, default to 1 page at most' },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -21,65 +22,105 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['www.uber.com/:language/blog/engineering', 'www.uber.com/:language/blog'],
+            source: ['www.uber.com/:language/blog/engineering'],
             target: '/blog',
         },
     ],
     name: 'Engineering',
     maintainers: ['hulb'],
     handler,
-    url: 'www.uber.com/blog/pittsburgh/engineering',
+    url: 'www.uber.com/en-HK/blog/engineering',
+    description:
+        "The English blog on any of Uber's regional sites (e.g., www.uber.com/en-JP/blog) is the same engineering blog provided by this route, so language selection is not supported. This route is not for the public news blog on specific regional sites (e.g., www.uber.com/ja-JP/blog).",
+    zh: {
+        description: 'uber的任何区域站点的英文blog（例如www.uber.com/en-JP/blog）都是相同的内容，正是本路由提供的engineering blog，因此本路由不提供语言选择；本路由不是uber在特定区域站点的公开新闻blog（例如www.uber.com/ja-JP/blog)',
+    },
 };
 
-async function handler(ctx) {
-    let maxPage = Number(ctx.req.param('maxPage'));
-    if (Number.isNaN(maxPage)) {
-        maxPage = 1;
-    }
-
-    let pages = await Promise.all(
-        [...Array(maxPage).keys()].map((pageIdx) =>
-            got(`${apiURL}/wp-json/blog/v1/data`, {
-                searchParams: {
-                    page: pageIdx + 1,
-                    parent: 'pittsburgh',
-                    slug: 'engineering',
-                },
-            })
-        )
-    );
-    pages = pages.map((page) => page.data);
+async function handler() {
+    const response = await ofetch(`${rootURL}/en-HK/blog/engineering/rss/`, {
+        // The source site is misconfigured or intentionally blocking requests without a specific accept header
+        // Without this header, it will return an HTTP 406 error
+        // Note that the accept type must be 'text/html'; 'application/xml' or similar will get HTTP 404 error
+        headers: {
+            accept: 'text/html',
+        },
+        // Without this, ofetch will parse the response as a blob instead of text, which cannot be loaded by cheerio
+        parseResponse: (txt) => txt,
+    });
+    const $ = load(response, { xmlMode: true });
 
     const result = await Promise.all(
-        pages.map((page) =>
-            Promise.all(
-                page.posts.map((post) =>
-                    cache.tryGet(`${rootURL}${post.link}`, async () => {
-                        let { data: article } = await got(`${apiURL}/wp-json/blog/v1/data`, {
-                            searchParams: {
-                                slug: post.link.replaceAll('/', '').replace('blog', ''),
-                            },
-                        });
-                        article = article.article;
+        $('item')
+            .toArray()
+            .map((el) =>
+                cache.tryGet($(el).find('link').text(), async () => {
+                    const detailResponse = await ofetch($(el).find('link').text(), {
+                        headers: {
+                            accept: 'text/html',
+                        },
+                    });
+                    const detail = load(detailResponse);
+
+                    const scriptText = detail('script#__REDUX_STATE__').text().trim();
+                    // The json in the script element is over-encoded
+                    // It needs to be decoded this way before it can be parsed by JSON.parse
+                    const jsonText = decodeURIComponent(JSON.parse(`"${scriptText}"`));
+                    // Traverse the JSON to find the content node, which is more robust against format changes.
+                    const contentHtml = findNode(JSON.parse(jsonText), { idKey: 'id', idValue: 'BlogArticleContent', siblingKey: 'props', childKey: 'content' }).replaceAll(String.raw`\n`, '');
 
-                        return {
-                            link: article.link,
-                            title: article.title,
-                            description: article.content,
-                            pubDate: parseDate(article.created),
-                            author: article.author,
-                            category: article.categories.map((category) => category.category_name),
-                        };
-                    })
-                )
+                    return {
+                        link: $(el).find('link').text(),
+                        title: $(el).find('title').text(),
+                        description: contentHtml,
+                        pubDate: parseDate($(el).find('pubDate').text()),
+                        category: $(el)
+                            .find('category')
+                            .toArray()
+                            .map((item) => $(item).text()),
+                    };
+                })
             )
-        )
     );
 
     return {
         title: `Uber Engineering Blog`,
         link: rootURL + '/blog/engineering',
         description: 'The technology behind Uber Engineering',
-        item: result.flat(),
+        item: result,
     };
 }
+
+function findNode(
+    json: any,
+    options: {
+        idKey?: string;
+        idValue: string;
+        siblingKey: string;
+        childKey: string;
+    }
+): any {
+    const { idKey = 'id', idValue, siblingKey, childKey } = options;
+
+    if (Array.isArray(json)) {
+        for (const item of json) {
+            const result = findNode(item, options);
+            if (result !== undefined) {
+                return result;
+            }
+        }
+    } else if (json && typeof json === 'object') {
+        if (json[idKey] === idValue) {
+            return json[siblingKey]?.[childKey];
+        }
+
+        for (const key in json) {
+            const result = findNode(json[key], options);
+            if (result !== undefined) {
+                return result;
+            }
+        }
+    }
+
+    return undefined;
+}

package/lib/routes/ucas/index.ts

@@ -47,7 +47,7 @@ async function handler(ctx) {
     const response = await got.get(url);
 
     const $ = load(response.data);
-    const list = $('#col1_content > div.list > ul > li').get();
+    const list = $('#col1_content > div.list > ul > li').toArray();
 
     const items = await Promise.all(
         list.map(async (item) => {

package/lib/routes/udn/breaking-news.ts

@@ -1,6 +1,4 @@
 import { Route } from '@/types';
-import { getCurrentPath } from '@/utils/helpers';
-const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
 import got from '@/utils/got';
@@ -55,7 +53,7 @@ async function handler(ctx) {
                 let result = await got(link);
                 // VIP article requires redirection
                 // e.g. https://udn.com/news/story/7331/6576320
-                const vip = result.data.match(/<script language=javascript>window\.location\.href="(https?:\/\/[^"]+")/);
+                const vip = result.data.match(/<script language=javascript>window\.location\.href="(https?:\/\/[^"]+)"/);
                 if (vip) {
                     result = await got(vip[1]);
                 }
@@ -85,7 +83,7 @@ async function handler(ctx) {
                     description += body.html();
                 }
 
-                if (data.publisher.name === '轉角國際 udn Global') {
+                if (data.publisher.name.includes('轉角國際 udn Global')) {
                     // 轉角24小時
                     description = $('.story_body_content')
                         .html()
@@ -96,7 +94,7 @@ async function handler(ctx) {
 
                 return {
                     title: item.title,
-                    author: data.author.name,
+                    author: [{ name: $('.article-content__author').text().match('中央社')?.at(0) }, { name: data.publisher.name.match('轉角國際 udn Global')?.at(0) }, data.author].filter((e) => Boolean(e.name)),
                     description,
                     pubDate: timezone(parseDate(item.time.date, 'YYYY-MM-DD HH:mm'), +8),
                     category: [data.articleSection, vip ? $('.article-head li.breadcrumb__item:last > b').text() : $("meta[name='subsection']").attr('content'), ...data.keywords.split(',')],
@@ -120,7 +118,7 @@ const getLinkName = async (link) => {
         const result = await got(url);
         const $ = load(result.data);
         const data = $('.cate-list__subheader a')
-            .get()
+            .toArray()
             .map((item) => {
                 item = $(item);
                 return [item.attr('href'), item.text().trim()];

package/lib/routes/udn/global/index.ts

@@ -20,25 +20,22 @@ export const route: Route = {
     },
     radar: [
         {
-            source: ['global.udn.com/global_vision/index/:category', 'global.udn.com/'],
+            source: ['global.udn.com/global_vision/index', 'global.udn.com/'],
         },
     ],
     name: '轉角國際 - 首頁',
     maintainers: ['nczitzk'],
     handler,
-    description: `| 首頁 | 最新文章 | 熱門文章 |
+    description: `| 首頁 | 編輯精選 | 熱門文章 |
 | ---- | -------- | -------- |
-|      | new      | hot      |`,
+|      | editor   | hot      |`,
 };
 
 async function handler(ctx) {
-    const category = ctx.req.param('category') ?? '';
-
-    const start = category === 'hot' ? 6 : 0;
-    const end = category === 'new' ? 6 : 12;
+    const category = ctx.req.param('category');
 
     const rootUrl = 'https://global.udn.com';
-    const currentUrl = `${rootUrl}/global_vision/index${category ? `/${category}` : ''}`;
+    const currentUrl = `${rootUrl}/global_vision/index`;
 
     const response = await got({
         method: 'get',
@@ -47,18 +44,28 @@ async function handler(ctx) {
 
     const $ = load(response.data);
 
-    $('.topic').remove();
-
-    let items = [...$('.news_cards ul li a').toArray().slice(start, end), ...(category === '' ? $('.last24, h2').find('a').toArray() : [])].map((item) => {
-        item = $(item);
-
-        const link = item.attr('href');
-
-        return {
-            title: item.find('h3').text() || item.text(),
-            link: link.startsWith('http') ? link : `${rootUrl}${item.attr('href')}`,
-        };
-    });
+    let articleSelector;
+    let titleExtractor;
+
+    if (category === 'hot') {
+        articleSelector = '.carousel__list .carousel__item';
+        titleExtractor = (element) => element.attr('title').trim();
+    } else {
+        const listContainer = category === 'editor' ? '.list-container--featured' : '.list-container--index';
+        articleSelector = `${listContainer} .list-vertical__item`;
+        titleExtractor = (element) => element.find('.list-vertical__title').text().trim();
+    }
+
+    let items = $(articleSelector)
+        .toArray()
+        .map((item) => {
+            const a = $(item);
+            const rawLink = a.attr('href').split('?')[0];
+            return {
+                title: titleExtractor(a),
+                link: rawLink.startsWith('http') ? rawLink : `${rootUrl}${rawLink}`,
+            };
+        });
 
     items = await Promise.all(
         items.map((item) =>
@@ -70,12 +77,17 @@ async function handler(ctx) {
 
                 const content = load(detailResponse.data);
 
-                content('#story_art_title, #story_bady_info, #story_also').remove();
-                content('.social_bar, .photo_pop, .only_mobile, .area').remove();
+                item.author = content('.article-content__authors-name').first().text().trim();
+                item.pubDate = timezone(parseDate(content('meta[property="article:published_time"]').attr('content')), +8);
+
+                const mainImage = content('.article-content__focus').html();
+                const articleBodyHtml = content('.article-content__editor')
+                    .find('p, figure, h2, .video-container')
+                    .toArray()
+                    .map((e) => content.html(e))
+                    .join('');
 
-                item.description = content('#tags').prev().html();
-                item.author = content('#story_author_name').text();
-                item.pubDate = timezone(parseDate(content('meta[name="date"]').attr('content')), +8);
+                item.description = mainImage + articleBodyHtml;
                 item.category = content('meta[name="news_keywords"]').attr('content').split(',');
 
                 return item;

package/lib/routes/udn/global/tag.ts

@@ -3,6 +3,7 @@ import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
 
 export const route: Route = {
     path: '/global/tag/:tag?',
@@ -42,11 +43,10 @@ async function handler(ctx) {
     });
 
     let items = response.data.articles.map((item) => ({
-        title: item.TITLE,
-        author: item.AUTHOR,
-        pubDate: parseDate(item.TIMESTAMP * 1000),
-        category: item.TAG.map((t) => t.tag),
-        link: `${rootUrl}/global_vision/story/${item.CATE_ID}/${item.ART_ID}`,
+        title: item.art_title,
+        // author: item.art_author_name,
+        // pubDate: timezone(parseDate(item.art_time), +8),
+        link: `${rootUrl}${item.link}`,
     }));
 
     items = await Promise.all(
@@ -59,11 +59,18 @@ async function handler(ctx) {
 
                 const content = load(detailResponse.data);
 
-                content('#story_art_title, #story_bady_info, #story_also').remove();
-                content('.social_bar, .photo_pop, .only_mobile, .area').remove();
+                item.author = content('.article-content__authors-name').first().text().trim();
+                item.pubDate = timezone(parseDate(content('meta[property="article:published_time"]').attr('content')), +8);
 
-                item.author = content('#story_author_name').text();
-                item.description = content('#tags').prev().html();
+                const mainImage = content('.article-content__focus').html();
+                const articleBodyHtml = content('.article-content__editor')
+                    .find('p, figure, h2, .video-container')
+                    .toArray()
+                    .map((e) => content.html(e))
+                    .join('');
+
+                item.description = mainImage + articleBodyHtml;
+                item.category = content('meta[name="news_keywords"]').attr('content').split(',');
 
                 return item;
             })

package/lib/routes/uestc/auto.ts

@@ -31,7 +31,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {
@@ -48,7 +48,8 @@ async function handler() {
     const items = $('dl.clearfix');
 
     const out = $(items)
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             const newsTitle = item.find('a').text();
             const newsLink = host + item.find('a[href]').attr('href').slice(3);
@@ -59,8 +60,7 @@ async function handler() {
                 link: newsLink,
                 pubDate: newsPubDate,
             };
-        })
-        .get();
+        });
 
     return {
         title: '电子科技大学自动化学院通知',

package/lib/routes/uestc/cqe.ts

@@ -51,7 +51,7 @@ async function handler(ctx) {
         throw new InvalidParameterError('type not supported');
     }
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {
@@ -68,7 +68,8 @@ async function handler(ctx) {
     const items = $('div.Newslist li');
 
     const out = $(items)
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             const newsTitle = item.find('a').attr('title');
             const newsLink = baseUrl + item.find('a').attr('href').slice(3);
@@ -79,8 +80,7 @@ async function handler(ctx) {
                 link: newsLink,
                 pubDate: newsPubDate,
             };
-        })
-        .get();
+        });
 
     return {
         title: `大学生文化素质教育中心-${mapTitle[type]}`,

package/lib/routes/uestc/news.ts

@@ -56,7 +56,8 @@ async function handler(ctx) {
     const items = $('div.notice-item.clearfix');
 
     const out = $(items)
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             const newsTitle = item.find('a').text().trim();
             const newsLink = baseUrl + item.find('a').attr('href');
@@ -69,8 +70,7 @@ async function handler(ctx) {
                 description: newsDescription,
                 pubDate: newsDate,
             };
-        })
-        .get();
+        });
 
     return {
         title: '新闻网通知',

package/lib/routes/uestc/scse.ts

@@ -45,7 +45,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {
@@ -74,7 +74,8 @@ async function handler() {
     });
 
     const out = $(items)
-        .map((index, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             const now = dayjs();
             let date = dayjs(now.year() + '-' + item.find('a span').text());
@@ -104,8 +105,7 @@ async function handler() {
                 link: newsLink,
                 pubDate: newsPubDate,
             };
-        })
-        .get();
+        });
 
     return {
         title: '计算机学院通知',

package/lib/routes/uestc/sice.ts

@@ -32,7 +32,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {
@@ -47,7 +47,8 @@ async function handler() {
     const $ = load(content);
 
     const out = $('.notice p')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             const now = dayjs();
             let date = dayjs(now.year() + '-' + item.find('a.date').text());
@@ -60,8 +61,7 @@ async function handler() {
                 link: host + item.find('a[href]').attr('href'),
                 pubDate: parseDate(date),
             };
-        })
-        .get();
+        });
 
     return {
         title: '信通学院通知',

package/lib/routes/uestc/sise.ts

@@ -66,7 +66,7 @@ async function handler(ctx) {
         throw new InvalidParameterError('type not supported');
     }
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {
@@ -83,7 +83,8 @@ async function handler(ctx) {
     const items = $(`div[id="${divId}"] p.news-item`);
 
     const out = $(items)
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             const now = dayjs();
             let date = dayjs(now.year() + '-' + item.find('span').text().replace('/', '-'));
@@ -99,8 +100,7 @@ async function handler(ctx) {
                 link: newsLink,
                 pubDate: newsPubDate,
             };
-        })
-        .get();
+        });
 
     return {
         title: `信软学院通知-${mapTitle[type]}`,

package/lib/routes/ulapia/index.ts

@@ -51,15 +51,15 @@ async function handler(ctx) {
     const response = await got.get(url);
     const $ = load(response.data);
     const items = $(String(selectorMap[category]))
-        .filter((_, item) => $(item).find('img').attr('src'))
-        .map((_, item) => ({
+        .toArray()
+        .filter((item) => $(item).find('img').attr('src'))
+        .map((item) => ({
             title: `${$(item).find('strong').text()}  ${$(item).find('h5.mb-1').text()}`,
             author: $(item).find('div.col.p-8.d-flex.px-3.py-3.flex-column.position-static > div:nth-child(4) > span:nth-child(2)').text(),
             link: $(item).find('h5.mb-1 > a').attr('href'),
             description: `<img src="${$(item).find('img').attr('src').split('!')[0]}">`,
             pubDate: parseDate($(item).find('div.mb-0.text-muted').last().text().split(':')[1], 'YYYY-MM-DD'),
-        }))
-        .get();
+        }));
 
     return {
         title: ` ulapia - ${titleMap[category]}`,

package/lib/routes/ulapia/research.ts

@@ -42,14 +42,14 @@ async function handler() {
                 const $ = load(response.data);
                 const items = $('div.row > div.col-md-6')
                     .slice(0, 6)
-                    .map((_, item) => ({
+                    .toArray()
+                    .map((item) => ({
                         title: `${$(item).find('strong').text()}  ${$(item).find('h5.mb-1').text()}`,
                         author: $(item).find('div.col.p-8.d-flex.px-3.py-3.flex-column.position-static > div:nth-child(4) > span:nth-child(2)').text(),
                         link: $(item).find('h5.mb-1 > a').attr('href'),
                         description: `<img src="${$(item).find('img').attr('src').split('!')[0]}">`,
                         pubDate: parseDate($(item).find('div.mb-0.text-muted').last().text().split(':')[1], 'YYYY-MM-DD'),
-                    }))
-                    .get();
+                    }));
 
                 return items;
             });