rsshub 1.0.0-master.f71451d → 1.0.0-master.f75997f

package/lib/config.ts

@@ -230,6 +230,9 @@ export type Config = {
     pkubbs: {
         cookie?: string;
     };
+    qingting: {
+        id?: string;
+    };
     saraba1st: {
         cookie?: string;
     };
@@ -258,6 +261,7 @@ export type Config = {
         oauthTokenSecrets?: string[];
         username?: string;
         password?: string;
+        authenticationSecret?: string;
         cookie?: string;
     };
     weibo: {
@@ -294,6 +298,9 @@ export type Config = {
     zodgame: {
         cookie?: string;
     };
+    zsxq: {
+        accessToken?: string;
+    };
 };
 
 const value: Config | Record<string, any> = {};
@@ -575,6 +582,9 @@ const calculateValue = () => {
         pkubbs: {
             cookie: envs.PKUBBS_COOKIE,
         },
+        qingting: {
+            id: envs.QINGTING_ID,
+        },
         saraba1st: {
             cookie: envs.SARABA1ST_COOKIE,
         },
@@ -607,6 +617,7 @@ const calculateValue = () => {
             oauthTokenSecrets: envs.TWITTER_OAUTH_TOKEN_SECRET?.split(','),
             username: envs.TWITTER_USERNAME,
             password: envs.TWITTER_PASSWORD,
+            authenticationSecret: envs.TWITTER_AUTHENTICATION_SECRET,
             cookie: envs.TWITTER_COOKIE,
         },
         weibo: {
@@ -643,6 +654,9 @@ const calculateValue = () => {
         zodgame: {
             cookie: envs.ZODGAME_COOKIE,
         },
+        zsxq: {
+            accessToken: envs.ZSXQ_ACCESS_TOKEN,
+        },
     };
 
     for (const name in _value) {

package/lib/errors/index.test.ts

@@ -66,13 +66,13 @@ describe('route throws an error', () => {
                     expect(value).toBe('9');
                     break;
                 case 'Hot Routes:':
-                    expect(value).toBe('6 /test/:id<br>');
+                    expect(value).toBe('6 /test/:id/:params?<br>');
                     break;
                 case 'Hot Paths:':
                     expect(value).toBe('2 /test/error<br>2 /test/slow<br>1 /test/httperror<br>1 /test/config-not-found-error<br>1 /test/invalid-parameter-error<br>1 /thisDoesNotExist<br>1 /<br>');
                     break;
                 case 'Hot Error Routes:':
-                    expect(value).toBe('5 /test/:id<br>');
+                    expect(value).toBe('5 /test/:id/:params?<br>');
                     break;
                 case 'Hot Error Paths:':
                     expect(value).toBe('2 /test/error<br>1 /test/httperror<br>1 /test/slow<br>1 /test/config-not-found-error<br>1 /test/invalid-parameter-error<br>1 /thisDoesNotExist<br>');

package/lib/middleware/template.tsx

@@ -1,4 +1,4 @@
-import { rss3Ums, json, RSS, Atom } from '@/utils/render';
+import { rss3, json, RSS, Atom } from '@/utils/render';
 import { config } from '@/config';
 import { collapseWhitespace, convertDateToISO8601 } from '@/utils/common-utils';
 import type { MiddlewareHandler } from 'hono';
@@ -48,6 +48,14 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
                     }
                 }
 
+                if (item.description) {
+                    // https://stackoverflow.com/questions/2507608/error-input-is-not-proper-utf-8-indicate-encoding-using-phps-simplexml-lo/40552083#40552083
+                    // https://stackoverflow.com/questions/1497885/remove-control-characters-from-php-string/1497928#1497928
+                    // remove unicode control characters
+                    // see #14940 #14943 #15262
+                    item.description = item.description.replaceAll(/[\u0000-\u0009\u000B\u000C\u000E-\u001F\u007F]/g, '');
+                }
+
                 if (typeof item.author === 'string') {
                     item.author = collapseWhitespace(item.author) || '';
                 } else if (typeof item.author === 'object' && item.author !== null) {
@@ -86,8 +94,9 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
         return ctx.json(result);
     }
 
-    if (outputType === 'ums') {
-        return ctx.json(rss3Ums(result));
+    // retain .ums for backward compatibility
+    if (outputType === 'ums' || outputType === 'rss3') {
+        return ctx.json(rss3(result));
     } else if (outputType === 'json') {
         ctx.header('Content-Type', 'application/feed+json; charset=UTF-8');
         return ctx.body(json(result));

package/lib/routes/0x80/index.ts

@@ -0,0 +1,87 @@
+import { Route } from '@/types';
+
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/blog',
+    categories: ['blog'],
+    example: '/0x80/blog',
+    url: '0x80.pl/notesen.html',
+    name: 'Articles',
+    maintainers: ['xnum'],
+    handler,
+};
+
+function extractDateFromURL(url: string) {
+    const regex = /\d{4}-\d{2}-\d{2}/;
+    const match = url.match(regex);
+
+    return match ? match[0] : null;
+}
+
+async function handler() {
+    // The TLS cert is invalid, we are limited to use HTTP unfortunately.
+    const baseUrl = 'http://0x80.pl/';
+    const targetUrl = `${baseUrl}notesen.html`;
+
+    const response = await got({
+        method: 'get',
+        url: targetUrl,
+    });
+
+    const $ = load(response.data);
+
+    const alist = $('a.reference.external');
+
+    const list = alist
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href') || '';
+            const title = item.text() || '';
+            const pubDate = extractDateFromURL(link);
+
+            return {
+                title,
+                link,
+                pubDate,
+                category: 'Uncategoried',
+            };
+        })
+        .filter((item) => item.link.startsWith('notesen'));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const articleUrl = `${baseUrl}${item.link}`;
+                const response = await got({
+                    method: 'get',
+                    url: articleUrl,
+                });
+
+                const $ = load(response.data);
+
+                const author = $('tr.author.field td.field-body').text();
+                const articlePubDate = $('tr.added-on.field td.field-body').text();
+
+                item.author = author;
+                // Some articles might be missing the added-on field.
+                // As a safeguard, if the date from url is null, fallbacks to the article one.
+                item.pubDate = parseDate(item.pubDate || articlePubDate);
+                item.description = $('div.document').first().html();
+
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: '0x80.pl articles',
+        link: targetUrl,
+        item: items,
+    };
+}

package/lib/routes/0x80/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Wojciech Muła',
+    url: '0x80.pl',
+    description: '',
+};

package/lib/routes/aljazeera/index.ts

@@ -4,11 +4,10 @@ const __dirname = getCurrentPath(import.meta.url);
 
 import { getSubPath } from '@/utils/common-utils';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
 import { load } from 'cheerio';
-import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
 import path from 'node:path';
+import { ofetch } from 'ofetch';
 
 const languages = {
     arabic: {
@@ -45,12 +44,8 @@ async function handler(ctx) {
     const rootUrl = languages[language].rootUrl;
     const currentUrl = `${rootUrl}/${isRSS ? languages[language].rssUrl : params.join('/')}`;
 
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-
-    const $ = load(response.data);
+    const response = await ofetch(currentUrl);
+    const $ = load(response);
 
     let items = isRSS
         ? response.data.match(new RegExp('<link>' + rootUrl + '/(.*?)</link>', 'g')).map((item) => ({
@@ -69,19 +64,27 @@ async function handler(ctx) {
     items = await Promise.all(
         items.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 50).map((item) =>
             cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
+                const detailResponse = await ofetch(item.link);
 
-                const content = load(detailResponse.data);
+                const content = load(detailResponse);
 
                 content('.more-on').parent().remove();
                 content('.responsive-image img').removeAttr('srcset');
+                let pubDate;
+
+                const datePublished = detailResponse.match(/"datePublished": ?"(.*?)",/);
+                if (datePublished && datePublished.length > 1) {
+                    pubDate = detailResponse.match(/"datePublished": ?"(.*?)",/)[1];
+                } else {
+                    // uploadDate replaces datePublished for video articles
+                    const uploadDate = detailResponse.match(/"uploadDate": ?"(.*?)",/)[1];
+
+                    pubDate = uploadDate && uploadDate.length > 1 ? uploadDate : content('div.date-simple > span:nth-child(2)').text();
+                }
 
                 item.title = content('h1').first().text();
                 item.author = content('.author').text();
-                item.pubDate = parseDate(detailResponse.data.match(/"datePublished": ?"(.*?)",/)[1]);
+                item.pubDate = pubDate;
                 item.description = art(path.join(__dirname, 'templates/description.art'), {
                     image: content('.article-featured-image').html(),
                     description: content('.wysiwyg').html(),

package/lib/routes/apple/podcast.ts

@@ -0,0 +1,64 @@
+import { Route } from '@/types';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/podcast/:id',
+    categories: ['multimedia'],
+    example: '/apple/podcast/id1559695855',
+    parameters: { id: '播客id，可以在 Apple 播客app 内分享的播客的 URL 中找到' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['podcasts.apple.com/cn/podcast/:id'],
+        },
+    ],
+    name: '播客',
+    maintainers: ['Acring'],
+    handler,
+    url: 'https://www.apple.com.cn/apple-podcasts/',
+};
+
+async function handler(ctx) {
+    const link = `https://podcasts.apple.com/cn/podcast/${ctx.req.param('id')}`;
+    const response = await got({
+        method: 'get',
+        url: link,
+    });
+
+    const $ = load(response.data);
+
+    const page_data = JSON.parse($('#shoebox-media-api-cache-amp-podcasts').text());
+
+    const data = JSON.parse(page_data[Object.keys(page_data)[0]]).d[0];
+    const attributes = data.attributes;
+
+    const episodes = data.relationships.episodes.data.map((item) => {
+        const attr = item.attributes;
+        return {
+            title: attr.name,
+            enclosure_url: attr.assetUrl,
+            itunes_duration: attr.durationInMilliseconds / 1000,
+            enclosure_type: 'audio/mp4',
+            link: attr.url,
+            pubDate: parseDate(attr.releaseDateTime),
+            description: attr.description.standard.replaceAll('\n', '<br>'),
+        };
+    });
+
+    return {
+        title: attributes.name,
+        link: attributes.url,
+        itunes_author: attributes.artistName,
+        item: episodes,
+        description: attributes.description.standard,
+    };
+}

package/lib/routes/bilibili/cache.ts

@@ -27,7 +27,7 @@ const getCookie = () => {
         await page.goto('https://space.bilibili.com/1/dynamic');
         const cookieString = await waitForRequest;
         logger.debug(`Got bilibili cookie: ${cookieString}`);
-
+        await browser.close();
         return cookieString;
     });
 };

package/lib/routes/bing/daily-wallpaper.ts

@@ -1,5 +1,7 @@
 import { Route } from '@/types';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
 
 export const route: Route = {
     path: '/',
@@ -9,25 +11,23 @@ export const route: Route = {
             target: '',
         },
     ],
-    name: 'Unknown',
+    name: '每日壁纸',
     maintainers: ['FHYunCai'],
     handler,
     url: 'cn.bing.com/',
 };
 
 async function handler(ctx) {
-    const response = await got({
-        method: 'get',
-        prefixUrl: 'https://cn.bing.com',
-        url: 'HPImageArchive.aspx',
-        searchParams: {
+    const response = await ofetch('HPImageArchive.aspx', {
+        baseURL: 'https://cn.bing.com',
+        query: {
             format: 'js',
             idx: 0,
             n: ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 7,
             mkt: 'zh-CN',
         },
     });
-    const data = response.data;
+    const data = response;
     return {
         title: 'Bing每日壁纸',
         link: 'https://cn.bing.com/',
@@ -35,6 +35,7 @@ async function handler(ctx) {
             title: item.copyright,
             description: `<img src="https://cn.bing.com${item.url}">`,
             link: item.copyrightlink,
+            pubDate: timezone(parseDate(item.fullstartdate), 0),
         })),
     };
 }

package/lib/routes/byau/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '黑龙江八一农垦大学',
+    url: 'byau.edu.cn',
+};

package/lib/routes/byau/xinwen/index.ts

@@ -0,0 +1,72 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+export const route: Route = {
+    path: '/news/:type_id',
+    categories: ['university'],
+    example: '/byau/news/3674',
+    parameters: { type_id: '栏目类型(从菜单栏获取对应 ID)' },
+    radar: [
+        {
+            source: ['xinwen.byau.edu.cn/:type_id/list.htm'],
+            target: '/news/:type_id',
+        },
+    ],
+    name: '新闻网',
+    maintainers: ['ueiu'],
+    handler,
+    url: 'xinwen.byau.edu.cn',
+    description: `| 学校要闻 | 校园动态 |
+  | ---- | ----------- |
+  | 3674 | 3676 |`,
+};
+
+async function handler(ctx) {
+    const baseUrl = 'http://xinwen.byau.edu.cn/';
+
+    const typeID = ctx.req.param('type_id');
+    const url = `${baseUrl}${typeID}/list.htm`;
+
+    const response = await got(url);
+    const $ = load(response.data);
+
+    const list = $('.news')
+        .toArray()
+        .map((item) => {
+            const $$ = load(item);
+
+            const originalItemUrl = $$('a').attr('href');
+            // 因为学校要闻的头两个像是固定了跳转专栏页面的，不能相同处理
+            const startsWithHttp = originalItemUrl.startsWith('http');
+            const itemUrl = startsWithHttp ? originalItemUrl : new URL(originalItemUrl, baseUrl).href;
+
+            return {
+                title: $$('a').text(),
+                link: itemUrl,
+                pubDate: timezone(parseDate($$('.news_meta').text()), +8),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = load(response.data);
+
+                item.description = $('.col_news_con').html();
+
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: $('title').text(),
+        link: url,
+        item: items,
+    };
+}