rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/alpinelinux/pkgs.ts

@@ -0,0 +1,104 @@
+import { Data, Route } from '@/types';
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+import { Context } from 'hono';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import { config } from '@/config';
+
+export const route: Route = {
+    name: 'Packages',
+    categories: ['program-update'],
+    maintainers: ['CaoMeiYouRen'],
+    path: '/pkgs/:name/:routeParams?',
+    parameters: { name: 'Packages name', routeParams: 'Filters of packages type. E.g. branch=edge&repo=main&arch=armv7&maintainer=Jakub%20Jirutka' },
+    example: '/alpinelinux/pkgs/nodejs',
+    description: `Alpine Linux packages update`,
+    handler,
+    radar: [
+        {
+            source: ['https://pkgs.alpinelinux.org/packages'],
+            target: (params, url) => {
+                const searchParams = new URL(url).searchParams;
+                const name = searchParams.get('name');
+                searchParams.delete('name');
+                const routeParams = searchParams.toString();
+                return `/alpinelinux/pkgs/${name}/${routeParams}`;
+            },
+        },
+    ],
+    zh: {
+        name: '软件包',
+        description: 'Alpine Linux 软件包更新',
+    },
+};
+
+type RowData = {
+    package: string;
+    packageUrl?: string;
+    version: string;
+    description?: string;
+    project?: string;
+    license: string;
+    branch: string;
+    repository: string;
+    architecture: string;
+    maintainer: string;
+    buildDate: string;
+};
+
+function parseTableToJSON(tableHTML: string) {
+    const $ = load(tableHTML);
+    const data: RowData[] = $('tbody tr')
+        .toArray()
+        .map((row) => ({
+            package: $(row).find('.package a').text().trim(),
+            packageUrl: $(row).find('.package a').attr('href')?.trim(),
+            description: $(row).find('.package a').attr('aria-label')?.trim(),
+            version: $(row).find('.version a').text().trim(),
+            project: $(row).find('.url a').attr('href')?.trim(),
+            license: $(row).find('.license').text().trim(),
+            branch: $(row).find('.branch').text().trim(),
+            repository: $(row).find('.repo a').text().trim(),
+            architecture: $(row).find('.arch a').text().trim(),
+            maintainer: $(row).find('.maintainer a').text().trim(),
+            buildDate: $(row).find('.bdate').text().trim(),
+        }));
+
+    return data;
+}
+
+async function handler(ctx: Context): Promise<Data> {
+    const { name, routeParams } = ctx.req.param();
+    const query = new URLSearchParams(routeParams);
+    query.append('name', name);
+    const link = `https://pkgs.alpinelinux.org/packages?${query.toString()}`;
+    const key = `alpinelinux:${query.toString()}`;
+    const rowData = (await cache.tryGet(
+        key,
+        async () => {
+            const response = await got({
+                url: link,
+            });
+            const html = response.data;
+            return parseTableToJSON(html);
+        },
+        config.cache.routeExpire,
+        false
+    )) as RowData[];
+
+    const items = rowData.map((e) => ({
+        title: `${e.package}@${e.version}/${e.architecture}`,
+        description: `Version: ${e.version}<br>Project: ${e.project}<br>Description: ${e.description}<br>License: ${e.license}<br>Branch: ${e.branch}<br>Repository: ${e.repository}<br>Maintainer: ${e.maintainer}<br>Build Date: ${e.buildDate}`,
+        link: `https://pkgs.alpinelinux.org${e.packageUrl}`,
+        guid: `https://pkgs.alpinelinux.org${e.packageUrl}#${e.version}`,
+        author: e.maintainer,
+        pubDate: parseDate(e.buildDate),
+    }));
+    return {
+        title: `${name} - Alpine Linux packages`,
+        link,
+        description: 'Alpine Linux packages update',
+        item: items,
+    };
+}

package/lib/routes/anime1/anime.ts

@@ -0,0 +1,63 @@
+import { Route } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+
+export const route: Route = {
+    path: 'anime/:category/:name',
+    name: 'Anime',
+    url: 'anime1.me',
+    maintainers: ['cxheng315'],
+    example: '/anime1/anime/2024年夏季/神之塔-第二季',
+    categories: ['anime'],
+    parameters: {
+        category: 'Anime1 Category',
+        name: 'Anime1 Name',
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['anime1.me/category/:category/:name'],
+            target: '/anime/:category/:name',
+        },
+    ],
+    handler,
+};
+
+async function handler(ctx) {
+    const { category, name } = ctx.req.param();
+
+    const response = await ofetch(`https://anime1.me/category/${category}/${name}`);
+
+    const $ = load(response);
+
+    const title = $('page-title').text().trim();
+
+    const items = $('article')
+        .toArray()
+        .map((el) => {
+            const $el = $(el);
+            const title = $el.find('.entry-title a').text().trim();
+            return {
+                title,
+                link: $el.find('.entry-title a').attr('href'),
+                description: title,
+                pubDate: parseDate($el.find('time').attr('datetime') || ''),
+                itunes_item_image: $el.find('video').attr('poster'),
+            };
+        });
+
+    return {
+        title,
+        link: `https://anime1.me/category/${category}/${name}`,
+        description: title,
+        item: items,
+    };
+}

package/lib/routes/anime1/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Anime1',
+    url: 'anime1.me',
+};

package/lib/routes/anime1/search.ts

@@ -0,0 +1,57 @@
+import { Route } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+
+export const route: Route = {
+    path: 'search/:keyword',
+    name: 'Search',
+    url: 'anime1.me',
+    maintainers: ['cxheng315'],
+    example: '/anime1/search/神之塔',
+    categories: ['anime'],
+    parameters: {
+        keyword: 'Anime1 Search Keyword',
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    handler,
+};
+
+async function handler(ctx) {
+    const { keyword } = ctx.req.param();
+
+    const response = await ofetch(`https://anime1.me/?s=${keyword}`);
+
+    const $ = load(response);
+
+    const title = $('page-title').text().trim();
+
+    const items = $('article.type-post')
+        .toArray()
+        .map((el) => {
+            const $el = $(el);
+            const title = $el.find('.entry-title a').text().trim();
+            return {
+                title,
+                link: $el.find('.entry-title a').attr('href'),
+                description: title,
+                pubDate: parseDate($el.find('time').attr('datetime') || ''),
+            };
+        });
+
+    return {
+        title,
+        link: `https://anime1.me/?s=${keyword}`,
+        description: title,
+        itunes_author: 'Anime1',
+        itunes_image: 'https://anime1.me/wp-content/uploads/2021/02/cropped-1-180x180.png',
+        item: items,
+    };
+}

package/lib/routes/apnews/api.ts

@@ -0,0 +1,62 @@
+import { Route, ViewType } from '@/types';
+import { fetchArticle } from './utils';
+import ofetch from '@/utils/ofetch';
+import timezone from '@/utils/timezone';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/api/:tags?',
+    categories: ['traditional-media'],
+    example: '/apnews/api/business',
+    view: ViewType.Articles,
+    parameters: {
+        tags: {
+            description: 'Getting a list of articles from a public API based on tags. See https://github.com/kovidgoyal/calibre/blob/81666219718b5f57d56b149a7ac017cc2a76b931/recipes/ap.recipe#L43-L46',
+            default: 'apf-topnews',
+        },
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['apnews.com/'],
+        },
+    ],
+    name: 'News',
+    maintainers: ['dzx-dzx'],
+    handler,
+};
+
+async function handler(ctx) {
+    const { tags = 'apf-topnews' } = ctx.req.param();
+    const apiRootUrl = 'https://afs-prod.appspot.com/api/v2/feed/tag';
+    const url = `${apiRootUrl}?tags=${tags}`;
+    const res = await ofetch(url);
+
+    const list = res.cards
+        .map((e) => ({
+            title: e.contents[0]?.headline,
+            link: e.contents[0]?.localLinkUrl,
+            pubDate: timezone(parseDate(e.publishedDate), 0),
+            category: e.tagObjs.map((tag) => tag.name),
+            updated: timezone(parseDate(e.contents[0]?.updated), 0),
+            description: e.contents[0]?.storyHTML,
+            author: e.contents[0]?.reporters.map((author) => ({ name: author.displayName })),
+        }))
+        .sort((a, b) => b.pubDate - a.pubDate)
+        .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20);
+
+    const items = ctx.req.query('mode') === 'fulltext' ? await Promise.all(list.map((item) => fetchArticle(item))) : list;
+
+    return {
+        title: `${res.tagObjs[0].name} - AP News`,
+        item: items,
+        link: 'https://apnews.com',
+    };
+}

package/lib/routes/apnews/rss.ts

@@ -0,0 +1,47 @@
+import { Route, ViewType } from '@/types';
+import parser from '@/utils/rss-parser';
+import { fetchArticle } from './utils';
+const HOME_PAGE = 'https://apnews.com';
+
+export const route: Route = {
+    path: '/rss/:category?',
+    categories: ['traditional-media', 'popular'],
+    example: '/apnews/rss/business',
+    view: ViewType.Articles,
+    parameters: {
+        category: {
+            description: 'Category from the first segment of the corresponding site, or `index` for the front page.',
+            default: 'index',
+        },
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['apnews.com/:rss'],
+            target: '/rss/:rss',
+        },
+    ],
+    name: 'News',
+    maintainers: ['zoenglinghou', 'mjysci', 'TonyRL'],
+    handler,
+};
+
+async function handler(ctx) {
+    const { rss = 'index' } = ctx.req.param();
+    const url = `${HOME_PAGE}/${rss}.rss`;
+    const res = await parser.parseURL(url);
+
+    const items = ctx.req.query('mode') === 'fulltext' ? await Promise.all(res.items.map((item) => fetchArticle(item))) : res;
+
+    return {
+        ...res,
+        item: items,
+    };
+}

package/lib/routes/apnews/sitemap.ts

@@ -0,0 +1,99 @@
+import { Route, ViewType } from '@/types';
+import { fetchArticle } from './utils';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import asyncPool from 'tiny-async-pool';
+import timezone from '@/utils/timezone';
+const HOME_PAGE = 'https://apnews.com';
+
+export const route: Route = {
+    path: '/sitemap/:route',
+    categories: ['traditional-media'],
+    example: '/apnews/sitemap/ap-sitemap-latest',
+    view: ViewType.Articles,
+    parameters: {
+        route: {
+            description: 'Route for sitemap, excluding the `.xml` extension',
+        },
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['apnews.com/'],
+        },
+    ],
+    name: 'Sitemap',
+    maintainers: ['zoenglinghou', 'mjysci', 'TonyRL', 'dzx-dzx'],
+    handler,
+};
+
+async function handler(ctx) {
+    const route = ctx.req.param('route');
+    const url = `${HOME_PAGE}/${route}.xml`;
+    const response = await ofetch(url);
+    const $ = load(response);
+
+    const list = $('urlset url')
+        .toArray()
+        .map((e) => {
+            const LANGUAGE_MAP = new Map([
+                ['eng', 'en'],
+                ['spa', 'es'],
+            ]);
+
+            const title = $(e)
+                .find(String.raw`news\:title`)
+                .text();
+            const pubDate = parseDate(
+                $(e)
+                    .find(String.raw`news\:publication_date`)
+                    .text()
+            );
+            const lastmod = timezone(parseDate($(e).find(`lastmod`).text()), -4);
+            const language = LANGUAGE_MAP.get(
+                $(e)
+                    .find(String.raw`news\:language`)
+                    .text()
+            );
+            let res = { link: $(e).find('loc').text() };
+            if (title) {
+                res = Object.assign(res, { title });
+            }
+            if (pubDate.toString() !== 'Invalid Date') {
+                res = Object.assign(res, { pubDate });
+            }
+            if (language) {
+                res = Object.assign(res, { language });
+            }
+            if (lastmod.toString() !== 'Invalid Date') {
+                res = Object.assign(res, { lastmod });
+            }
+            return res;
+        })
+        .filter((e) => Boolean(e.link) && !new URL(e.link).pathname.split('/').includes('hub'))
+        .sort((a, b) => (a.pubDate && b.pubDate ? b.pubDate - a.pubDate : b.lastmod - a.lastmod))
+        .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20);
+
+    const items = ctx.req.query('mode') === 'fulltext' ? await asyncPoolAll(20, list, (item) => fetchArticle(item)) : list;
+
+    return {
+        title: `AP News sitemap:${route}`,
+        item: items,
+        link: 'https://apnews.com',
+    };
+}
+async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
+    const results: Awaited<OUT[]> = [];
+    for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
+        results.push(result);
+    }
+    return results;
+}

package/lib/routes/apnews/topics.ts

@@ -1,15 +1,20 @@
-import { Route } from '@/types';
-import cache from '@/utils/cache';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
-import timezone from '@/utils/timezone';
+import { fetchArticle, removeDuplicateByKey } from './utils';
 const HOME_PAGE = 'https://apnews.com';
 
 export const route: Route = {
     path: '/topics/:topic?',
-    categories: ['traditional-media'],
+    categories: ['traditional-media', 'popular'],
     example: '/apnews/topics/apf-topnews',
-    parameters: { topic: 'Topic name, can be found in URL. For example: the topic name of AP Top News [https://apnews.com/apf-topnews](https://apnews.com/apf-topnews) is `apf-topnews`, `trending-news` by default' },
+    view: ViewType.Articles,
+    parameters: {
+        topic: {
+            description: 'Topic name, can be found in URL. For example: the topic name of AP Top News [https://apnews.com/apf-topnews](https://apnews.com/apf-topnews) is `apf-topnews`',
+            default: 'trending-news',
+        },
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -37,34 +42,21 @@ async function handler(ctx) {
 
     const items = await Promise.all(
         $(':is(.PagePromo-content, .PageListStandardE-leadPromo-info) bsp-custom-headline')
-            .get()
+            .toArray()
             .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : Infinity)
             .map((e) => ({
                 title: $(e).find('span.PagePromoContentIcons-text').text(),
                 link: $(e).find('a').attr('href'),
             }))
             .filter((e) => typeof e.link === 'string')
-            .map((item) =>
-                cache.tryGet(item.link, async () => {
-                    const { data: response } = await got(item.link);
-                    const $ = load(response);
-                    $('div.Enhancement').remove();
-                    return Object.assign(item, {
-                        pubDate: timezone(new Date($("meta[property='article:published_time']").attr('content')), 0),
-                        updated: timezone(new Date($("meta[property='article:modified_time']").attr('content')), 0),
-                        description: $('div.RichTextStoryBody').html(),
-                        category: $("meta[property='article:section']").attr('content'),
-                        guid: $("meta[name='brightspot.contentId']").attr('content'),
-                    });
-                })
-            )
+            .map((item) => (ctx.req.query('mode') === 'fulltext' ? fetchArticle(item) : item))
     );
 
     return {
         title: $('title').text(),
         description: $("meta[property='og:description']").text(),
         link: url,
-        item: items,
+        item: removeDuplicateByKey(items, 'guid'),
         language: $('html').attr('lang'),
     };
 }

package/lib/routes/apnews/utils.ts

@@ -0,0 +1,48 @@
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+
+export function removeDuplicateByKey(items, key: string) {
+    return [...new Map(items.map((x) => [x[key], x])).values()];
+}
+
+export function fetchArticle(item) {
+    return cache.tryGet(item.link, async () => {
+        const data = await ofetch(item.link);
+        const $ = load(data);
+        if ($('#link-ld-json').length === 0) {
+            return item;
+        }
+        const rawLdjson = JSON.parse($('#link-ld-json').text());
+        let ldjson;
+        if (rawLdjson['@type'] === 'NewsArticle' || (Array.isArray(rawLdjson) && rawLdjson.some((e) => e['@type'] === 'NewsArticle'))) {
+            // Regular(Articles, Videos)
+            ldjson = Array.isArray(rawLdjson) ? rawLdjson.find((e) => e['@type'] === 'NewsArticle') : rawLdjson;
+
+            $('div.Enhancement').remove();
+            const section = $("meta[property='article:section']").attr('content');
+            return {
+                title: ldjson.headline,
+                pubDate: parseDate(ldjson.datePublished),
+                updated: parseDate(ldjson.dateModified),
+                description: $('div.RichTextStoryBody').html() || $(':is(.VideoLead, .VideoPage-pageSubHeading)').html(),
+                category: [...(section ? [section] : []), ...(ldjson.keywords ?? [])],
+                guid: $("meta[name='brightspot.contentId']").attr('content'),
+                author: ldjson.author,
+                ...item,
+            };
+        } else {
+            // Live
+            ldjson = rawLdjson;
+
+            return {
+                category: ldjson.keywords,
+                pubDate: parseDate(ldjson.coverageStartTime),
+                description: ldjson.description,
+                guid: $("meta[name='brightspot.contentId']").attr('content'),
+                ...item,
+            };
+        }
+    });
+}

package/lib/routes/apple/apps.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
@@ -17,9 +17,34 @@ const platforms = {
 
 export const route: Route = {
     path: '/apps/update/:country/:id/:platform?',
-    categories: ['program-update'],
+    categories: ['program-update', 'popular'],
+    view: ViewType.Notifications,
     example: '/apple/apps/update/us/id408709785',
-    parameters: { country: 'App Store Country, obtain from the app URL, see below', id: 'App id, obtain from the app URL', platform: 'App Platform, see below, all by default' },
+    parameters: {
+        country: 'App Store Country, obtain from the app URL, see below',
+        id: 'App id, obtain from the app URL',
+        platform: {
+            description: 'App Platform, see below, all by default',
+            options: [
+                {
+                    value: 'All',
+                    label: 'all',
+                },
+                {
+                    value: 'iOS',
+                    label: 'iOS',
+                },
+                {
+                    value: 'macOS',
+                    label: 'macOS',
+                },
+                {
+                    value: 'tvOS',
+                    label: 'tvOS',
+                },
+            ],
+        },
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -37,10 +62,7 @@ export const route: Route = {
     name: 'App Update',
     maintainers: ['EkkoG', 'nczitzk'],
     handler,
-    description: `| All | iOS | macOS | tvOS |
-  | --- | --- | ----- | ---- |
-  |     | iOS | macOS | tvOS |
-
+    description: `
   :::tip
   For example, the URL of [GarageBand](https://apps.apple.com/us/app/messages/id408709785) in the App Store is \`https://apps.apple.com/us/app/messages/id408709785\`. In this case, the \`App Store Country\` parameter for the route is \`us\`, and the \`App id\` parameter is \`id1146560473\`. So the route should be [\`/apple/apps/update/us/id408709785\`](https://rsshub.app/apple/apps/update/us/id408709785).
   :::`,
@@ -52,7 +74,7 @@ async function handler(ctx) {
 
     let platformId;
 
-    if (platform) {
+    if (platform && platform !== 'all') {
         platform = platform.toLowerCase();
         platformId = Object.hasOwn(platforms, platform) ? platforms[platform] : platform;
     }
@@ -123,7 +145,7 @@ async function handler(ctx) {
 
     return {
         item: items,
-        title: `${title} - Apple App Stroe`,
+        title: `${title} - Apple App Store`,
         link: currentUrl,
         description: description?.replace(/\n/g, ' '),
         language: $('html').prop('lang'),