rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa04a11

package/lib/routes/discord/search.ts

@@ -0,0 +1,106 @@
+import path from 'node:path';
+
+import { config } from '@/config';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+import { Route } from '@/types';
+import { getCurrentPath } from '@/utils/helpers';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import { queryToBoolean } from '@/utils/readable-social';
+
+import { baseUrl, getGuild, searchGuildMessages, SearchGuildMessagesParams, HasType, VALID_HAS_TYPES } from './discord-api';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+export const route: Route = {
+    path: '/search/:guildId/:routeParams',
+    categories: ['social-media'],
+    example: '/discord/search/302094807046684672/content=friendly&has=image,video',
+    parameters: {
+        guildId: 'Guild ID',
+        routeParams: 'Search parameters, support content, author_id, mentions, has, min_id, max_id, channel_id, pinned',
+    },
+    features: {
+        requireConfig: [
+            {
+                name: 'DISCORD_AUTHORIZATION',
+                description: 'Discord authorization header',
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Guild Search',
+    maintainers: ['NekoAria'],
+    handler,
+};
+
+const parseSearchParams = (routeParams?: string): SearchGuildMessagesParams => {
+    const parsed = new URLSearchParams(routeParams);
+    const hasTypes = parsed.get('has')?.split(',').filter(Boolean);
+    const validHasTypes = hasTypes?.filter((type) => VALID_HAS_TYPES.has(type as HasType)) as HasType[];
+
+    const params = {
+        content: parsed.get('content') ?? undefined,
+        author_id: parsed.get('author_id') ?? undefined,
+        mentions: parsed.get('mentions') ?? undefined,
+        has: validHasTypes?.length ? validHasTypes : undefined,
+        min_id: parsed.get('min_id') ?? undefined,
+        max_id: parsed.get('max_id') ?? undefined,
+        channel_id: parsed.get('channel_id') ?? undefined,
+        pinned: parsed.has('pinned') ? queryToBoolean(parsed.get('pinned')) : undefined,
+    };
+
+    return Object.fromEntries(Object.entries(params).filter(([, value]) => value !== undefined));
+};
+
+async function handler(ctx) {
+    const { authorization } = config.discord || {};
+    if (!authorization) {
+        throw new ConfigNotFoundError('Discord RSS is disabled due to the lack of authorization config');
+    }
+
+    const { guildId } = ctx.req.param();
+    const searchParams = parseSearchParams(ctx.req.param('routeParams'));
+
+    if (!Object.keys(searchParams).length) {
+        throw new InvalidParameterError('At least one valid search parameter is required');
+    }
+
+    const [guildInfo, searchResult] = await Promise.all([getGuild(guildId, authorization), searchGuildMessages(guildId, authorization, searchParams)]);
+
+    if (!searchResult?.messages?.length) {
+        return {
+            title: `Search Results - ${guildInfo.name}`,
+            link: `${baseUrl}/channels/${guildId}`,
+            item: [],
+            allowEmpty: true,
+        };
+    }
+
+    const messages = searchResult.messages.flat().map((message) => ({
+        title: message.content.split('\n')[0] || '(no content)',
+        description: art(path.join(__dirname, 'templates/message.art'), { message, guildInfo }),
+        author: message.author.global_name ?? message.author.username,
+        pubDate: parseDate(message.timestamp),
+        updated: message.edited_timestamp ? parseDate(message.edited_timestamp) : undefined,
+        category: [`#${message.channel_id}`],
+        link: `${baseUrl}/channels/${guildId}/${message.channel_id}/${message.id}`,
+    }));
+
+    const searchDesc = Object.entries(searchParams)
+        .filter(([, value]) => value !== undefined)
+        .map(([key, value]) => `${key}:${Array.isArray(value) ? value.join(',') : value}`)
+        .join(' ');
+
+    return {
+        title: `Search "${searchDesc}" in ${guildInfo.name} - Discord`,
+        link: `${baseUrl}/channels/${guildId}`,
+        item: messages,
+        allowEmpty: true,
+    };
+}

package/lib/routes/discourse/namespace.ts

@@ -2,7 +2,7 @@ import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
     name: 'Discourse',
-    description: `:::warning
+    description: `::: warning
 You need to set the environment variable \`DISCOURSE_CONFIG_{id}\` before using it. Please refer to Configuration section in the Deploy page of the documentation.
 :::`,
     lang: 'en',

package/lib/routes/discourse/notifications.ts

@@ -25,7 +25,7 @@ export const route: Route = {
     name: 'Notifications',
     maintainers: [],
     handler,
-    description: `:::warning
+    description: `::: warning
 If you opt to enable \`fulltext\` feature, consider adding \`limit\` parameter to your query to avoid sending too many request.
 :::`,
 };

package/lib/routes/dlnews/category.ts

@@ -69,10 +69,11 @@ export const route: Route = {
             target: '/:category',
         },
     ],
-    name: 'Unknown',
+    url: 'dlnews.com/articles',
+    name: 'Latest News',
     maintainers: ['Rjnishant530'],
     handler,
-    url: 'dlnews.com/articles/',
+    example: '/dlnews/people-culture',
 };
 
 async function handler(ctx) {

package/lib/routes/dn/news.ts

@@ -12,7 +12,7 @@ import path from 'node:path';
 
 export const route: Route = {
     path: '/:language/news/:category?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/dn/en-us/news',
     parameters: { language: 'Language, see below', category: 'Category, see below, The Latest by default' },
     features: {

package/lib/routes/dnaindia/topic.ts

@@ -22,7 +22,7 @@ export const route: Route = {
 | ------------ |
 | dna-verified |
 
-:::tip
+::: tip
 The URL of the form \`https://www.dnaindia.com/topic/dna-verified\` demonstrates the utilization of the subdomain \`topic\`.
 :::`,
 };

package/lib/routes/dockerhub/repositories.ts

@@ -0,0 +1,42 @@
+import { Route, ViewType } from '@/types';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import { Context } from 'hono';
+
+export const route: Route = {
+    name: 'Owner Repositories',
+    description: 'List of repositories for an image owner',
+    maintainers: ['CaoMeiYouRen'],
+    path: '/repositories/:owner',
+    categories: ['program-update'],
+    view: ViewType.Notifications,
+    example: '/dockerhub/repositories/diygod',
+    parameters: { owner: 'Image owner' },
+    handler,
+};
+
+async function handler(ctx: Context) {
+    const owner = ctx.req.param('owner').toLowerCase();
+    const limit = Number.parseInt(ctx.req.query('limit') || '10');
+    const link = `https://hub.docker.com/r/${owner}`;
+    const url = `https://hub.docker.com/v2/repositories/${owner}`;
+    const response = await got(url, {
+        searchParams: {
+            page_size: limit,
+        },
+    });
+    const item = response.data.results.map((repo) => ({
+        title: repo.name,
+        description: `${repo.description}<br>status: ${repo.status_description}<br>stars: ${repo.star_count}<br>pulls: ${repo.pull_count}`,
+        link: `https://hub.docker.com/r/${owner}/${repo.name}`,
+        author: owner,
+        pubDate: parseDate(repo.last_updated),
+        guid: `${owner}/${repo.name}`,
+    }));
+    return {
+        title: `${owner} repositories`,
+        description: `List of repositories for ${owner}`,
+        link,
+        item,
+    };
+}

package/lib/routes/dockerhub/tag.ts

@@ -17,11 +17,11 @@ export const route: Route = {
         supportScihub: false,
     },
     name: 'Image New Tag',
-    maintainers: [],
+    maintainers: ['pseudoyu'],
     handler,
-    description: `:::warning
+    description: `::: warning
   Use \`library\` as the \`owner\` for official images, such as [https://rsshub.app/dockerhub/tag/library/mysql](https://rsshub.app/dockerhub/tag/library/mysql)
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {
@@ -42,14 +42,20 @@ async function handler(ctx) {
         description: metadata.data.description,
         link,
         language: 'en',
-        item: tags.map((item) => ({
-            title: `${namespace}:${item.name} was updated`,
-            description: `${namespace}:${item.name} was updated, supporting the architectures of ${item.images.map((img) => `${img.os}/${img.architecture}`).join(', ')}`,
-            link: `https://hub.docker.com/layers/${owner === 'library' ? `${image}/` : ''}${namespace}/${item.name}/images/${item.images[0].digest.replace(':', '-')}`,
-            author: owner,
-            pubDate: parseDate(item.tag_last_pushed),
-            // check for (1) different tag names and (2) different image hashes, considering varients of all arches
-            guid: `${namespace}:${item.name}@${hash(item.images)}`,
-        })),
+        item: tags.map((item) => {
+            const architectures = item.images?.length ? item.images.map((img) => `${img.os}/${img.architecture}`).join(', ') : 'unknown architectures';
+
+            const imageDigest = item.digest?.replace(':', '-') || '';
+            const layerLink = `https://hub.docker.com/layers/${owner === 'library' ? `${image}/` : ''}${namespace}/${item.name}/images/${imageDigest}`;
+
+            return {
+                title: `${namespace}:${item.name} was updated`,
+                description: `${namespace}:${item.name} was updated, supporting the ${architectures}`,
+                link: layerLink,
+                author: owner,
+                pubDate: parseDate(item.tag_last_pushed),
+                guid: `${namespace}:${item.name}@${hash(item.images || [])}`,
+            };
+        }),
     };
 }

package/lib/routes/domp4/detail.ts

@@ -91,7 +91,7 @@ export const route: Route = {
     name: '剧集订阅',
     maintainers: ['savokiss'],
     handler,
-    description: `:::tip
+    description: `::: tip
 由于大部分详情页是 \`/html/xxx.html\`，还有部分是 \`/detail/123.html\`，所以此处做了兼容，id 取 \`xxx\` 或者 \`123\` 都可以。
 
 新增 \`second\` 参数，用于选择下载地址二（地址二不可用或者不填都默认地址一），用法: \`/domp4/detail/LBTANI22222I?second=1\`。

package/lib/routes/domp4/namespace.ts

@@ -3,7 +3,7 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: 'DoMP4 影视',
     url: 'domp4.cc',
-    description: `:::tip
+    description: `::: tip
 由于网站有多个备用域名，默认使用 \`mp4us.com\`，可以通过在路由最后加上 \`?domain=<域名>\` 切换域名。
 
 目前可用域名有：

package/lib/routes/dongqiudi/daily.ts

@@ -1,7 +1,7 @@
 import { Route } from '@/types';
 export const route: Route = {
     path: '/daily',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/dongqiudi/daily',
     radar: [
         {
@@ -12,11 +12,11 @@ export const route: Route = {
     maintainers: ['HenryQW'],
     handler,
     url: 'www.dongqiudi.com/special/48',
-    description: `:::tip
+    description: `::: tip
 部分球队和球员可能会有两个 id, 正确 id 应该由 \`5000\` 开头.
 :::`,
 };
 
 function handler(ctx) {
-    ctx.redirect('/dongqiudi/special/48');
+    ctx.set('redirect', '/dongqiudi/special/48');
 }

package/lib/routes/dongqiudi/namespace.ts

@@ -3,7 +3,7 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: '懂球帝',
     url: 'm.dongqiudi.com',
-    description: `:::tip
+    description: `::: tip
 -   可以通过头条新闻 + 参数过滤的形式获得早报、专题等内容。
 :::`,
     lang: 'zh-CN',

package/lib/routes/dongqiudi/player-news.ts

@@ -3,7 +3,7 @@ import utils from './utils';
 
 export const route: Route = {
     path: '/player_news/:id',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/dongqiudi/player_news/50000339',
     parameters: { id: '球员 id, 可在[懂球帝数据](https://www.dongqiudi.com/data)中通过其队伍找到' },
     radar: [

package/lib/routes/dongqiudi/result.ts

@@ -5,7 +5,7 @@ import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/result/:team',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/dongqiudi/result/50001755',
     parameters: { team: '球队 id, 可在[懂球帝数据](https://www.dongqiudi.com/data)中找到' },
     radar: [

package/lib/routes/dongqiudi/special.ts

@@ -6,7 +6,7 @@ import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/special/:id',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/dongqiudi/special/41',
     parameters: { id: '专题 id, 可自行通过 https://www.dongqiudi.com/special/+数字匹配' },
     radar: [

package/lib/routes/dongqiudi/team-news.ts

@@ -3,7 +3,7 @@ import utils from './utils';
 
 export const route: Route = {
     path: '/team_news/:team',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/dongqiudi/team_news/50001755',
     parameters: { team: '球队 id, 可在[懂球帝数据](https://www.dongqiudi.com/data)中找到' },
     radar: [

package/lib/routes/dongqiudi/top-news.ts

@@ -6,7 +6,7 @@ import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/top_news/:id?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/dongqiudi/top_news/1',
     parameters: { id: '类别 id，不填默认头条新闻' },
     features: {

package/lib/routes/douban/other/recommended.ts

@@ -30,9 +30,9 @@ export const route: Route = {
 
   用例：\`/douban/recommended/tv/playable=0&score=8\`
 
-  :::tip
+::: tip
   整合了 /douban/list/ 路由，省去每月手动更新 id 参数，因为当月推荐剧集片单中，会有还未播出 / 开评分剧集、海外平台播出剧集，请自行考虑是否使用额外参数。
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/douyin/namespace.ts

@@ -3,7 +3,7 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: '抖音直播',
     url: 'douyin.com',
-    description: `:::warning
+    description: `::: warning
 反爬严格，需要启用 puppeteer。\
 抖音的视频 CDN 会验证 Referer，意味着许多阅读器都无法直接播放内嵌视频，以下是一些变通解决方案：
 

package/lib/routes/duckdb/news.ts

@@ -36,8 +36,9 @@ async function handler() {
             item = $(item);
             return {
                 title: item.find('h3').text().trim(),
-                link: `https://duckdb.org${item.find('a').eq(2).attr('href')}`,
+                link: `https://duckdb.org${item.find('a.blocklink').attr('href')}`,
                 pubDate: timezone(parseDate(item.find('.date').text(), 'YYYY-MM-DD'), 0),
+                author: item.find('.author').text().trim(),
             };
         });
 
@@ -46,9 +47,7 @@ async function handler() {
             cache.tryGet(item.link, async () => {
                 const response = await got(item.link);
                 const $ = load(response.body);
-                item.author = $('.author').text();
-                item.description = $('.singleentry').html();
-
+                item.description = $('.contentwidth').find('h1, .infoline').remove().end().html();
                 // 上面每个列表项的每个属性都在此重用，
                 // 并增加了一个新属性“description”
                 return item;

package/lib/routes/dw/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'DW Deutsche Welle',
+    url: 'dw.com',
+};

package/lib/routes/dw/news.ts

@@ -0,0 +1,89 @@
+import { Route } from '@/types';
+import { processItems } from './utils';
+import got from '@/utils/got';
+import cache from '@/utils/cache';
+import { config } from '@/config';
+
+export const route: Route = {
+    path: '/news/:lang?/:id?',
+    categories: ['traditional-media'],
+    example: '/dw/news',
+    parameters: {
+        lang: 'Language, see below, default to en',
+        id: 'Category ID, see below, default to the id of the Top Stories Page of the language chosen',
+    },
+    features: {
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+        requireConfig: false,
+    },
+    name: 'News',
+    maintainers: ['quiniapiezoelectricity'],
+    handler,
+    description: `
+::: tip
+Parameters can be obtained from the official website, for instance:
+For the site https://www.dw.com/de/deutschland/s-12321 the language code would be \`de\` and the category ID would be \`s-1432\`.
+:::
+`,
+    radar: [
+        {
+            source: ['www.dw.com/:lang/:name/:id'],
+            target: '/news/:lang/:id',
+        },
+    ],
+};
+
+const defaultUrl = `https://www.dw.com/graph-api/en/content/navigation/9097`;
+const typenames = new Set(['Article', 'Liveblog', 'Video']);
+
+async function handler(ctx) {
+    const lang = ctx.req.param('lang') ?? 'en';
+    let id = ctx.req.param('id');
+
+    if (/^s-\d+$/.test(id)) {
+        id = id.match(/^s-(\d+)$/i)[1]; // convert s-1234 id to 1234
+    } else if (id === undefined) {
+        // Look up the id of the Top Stories Page of the selected language if id is not specified in the URL.
+        const navigation = await cache.tryGet(
+            'dw:navigation',
+            async () => {
+                const res = await got(defaultUrl);
+                return res.data.data.content.topStoriesNavigations;
+            },
+            config.cache.routeExpire,
+            false
+        );
+        id = navigation
+            .map((item) => item.namedUrl.split('/'))
+            .find((item) => item[1] === lang)[3]
+            .match(/^s-(\d+)$/i)[1];
+    }
+
+    const response = await got(`https://www.dw.com/graph-api/${lang}/content/navigation/${id}`);
+    const feed = response.data.data.content;
+    cache.set('dw:navigation', feed.topStoriesNavigations, config.cache.routeExpire);
+
+    const list = feed.contentComposition.informationSpaces.flatMap((section) => Object.values(section).flatMap((component) => component[0]?.contents || [])).filter((item) => typenames.has(item.__typename) && item.id);
+    const items = await processItems(
+        list.map((item) => {
+            item.link = new URL(item.namedUrl, 'https://www.dw.com').href;
+            item.pubDate = item.contentDate;
+            item.description = item.teaser;
+            item.language = lang;
+            item.type = item.__typename.toLowerCase();
+            return item;
+        })
+    );
+
+    return {
+        title: `DW | ${feed.title}`,
+        link: feed.canonicalUrl,
+        description: feed.metaDescription,
+        language: feed.topStoriesNavigations.find((item) => item.namedUrl.startsWith(`/${lang}/`))?.localeLang ?? lang,
+        item: items,
+    };
+}

package/lib/routes/dw/rss.ts

@@ -0,0 +1,62 @@
+import { Route } from '@/types';
+import { config } from '@/config';
+import Parser from 'rss-parser';
+import { processItems } from './utils';
+
+export const route: Route = {
+    path: '/rss/:channel?',
+    categories: ['traditional-media'],
+    example: '/dw/rss/rss-en-all',
+    parameters: {
+        category: 'RSS Feed Channel, see below, `rss-en-all` by default',
+    },
+    features: {
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+        requireConfig: false,
+    },
+    name: 'RSS',
+    maintainers: ['quiniapiezoelectricity'],
+    handler,
+    description: `
+For a full list of RSS Feed Channels in English, please refer to [DW RSS Feeds](https://corporate.dw.com/en/rss-feeds/a-68693346).
+RSS Feed Channels in other languages are also available, for example: \`rss-chi-all\` renders the RSS feed in Chinese and \`rss-de-all\` for the RSS Feed in German 
+`,
+};
+
+async function handler(ctx) {
+    const category = ctx.req.param('channel') ?? 'rss-en-all';
+
+    const parser = new Parser({
+        customFields: {
+            item: ['dwsyn:contentID'],
+        },
+        headers: {
+            'User-Agent': config.ua,
+        },
+    });
+
+    const feed = await parser.parseURL(`https://rss.dw.com/rdf/${category}`);
+    const items = await processItems(
+        feed.items.map((item) => {
+            item.id = item['dwsyn:contentID'];
+            item.pubDate = item.isoDate;
+            item.description = item.content;
+            const link = new URL(item.link);
+            link.search = '';
+            item.link = link.href;
+            item.type = link.pathname.substring(link.pathname.lastIndexOf('/') + 1).startsWith('live-') ? 'liveblog' : 'article'; // dw rss feed only includes liveblogs and articles
+            return item;
+        })
+    );
+
+    return {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: items,
+    };
+}

package/lib/routes/dw/templates/description.art

@@ -0,0 +1,23 @@
+{{ if teaser }}
+    <blockquote><p><em>{{ teaser }}</em></p></blockquote>
+{{ /if }}
+{{ if video }}
+    {{@ video }}
+{{ else if mainImage }}
+    <figure>
+        <img 
+            src="https://static.dw.com/image/{{ mainImage.targetId }}_{{ formatId }}.jpg" 
+            alt="{{ mainImage.additionalInformation }}" 
+        >
+        <figcaption>
+            {{ mainImage.description }}
+            <small>{{ imageI18n }}: {{ mainImage.target.licenserSupplement }}</small>
+        </figcaption>
+    </figure>
+{{ /if }}
+{{ if text }}
+   {{@ text }}
+{{ /if }}
+{{ if liveblog }}
+    {{@ liveblog }}
+{{ /if }}

package/lib/routes/dw/templates/liveblog.art

@@ -0,0 +1,13 @@
+{{ if posts }}
+{{ each posts }}
+    <hr>
+    {{ if $value.localizedContentDate }}<p><i>{{ $value.localizedContentDate }}</i></p>{{ /if }}
+    {{ if $value.title }}<h2>{{ $value.title }}</h2>{{ /if }}
+    {{ if $value.persons }}
+        {{ each $value.persons }}
+        <p><i>{{ $value.fullName }}</i></p>
+        {{ /each }}
+    {{ /if }}
+    {{ if $value.text }}{{@ $value.text }}{{ /if }}
+{{ /each }}
+{{ /if }}

package/lib/routes/dw/templates/video.art

@@ -0,0 +1,14 @@
+{{ if hlsVideoSrc }}
+<video
+{{ if posterImageUrl }} poster="{{ posterImageUrl }}" {{ /if }}
+controls preload="metadata">
+    <source
+    src="{{ hlsVideoSrc }}"
+    type="application/x-mpegURL"
+    >
+    <source
+    src="{{ mp4VideoSrc }}"
+    type="video/mp4"
+    >
+</video>
+{{ /if }}