rsshub 1.0.0-master.ff487f3 → 1.0.0-master.ff76aa7

package/lib/routes/aibase/util.ts

@@ -0,0 +1,114 @@
+import { getCurrentPath } from '@/utils/helpers';
+const __dirname = getCurrentPath(import.meta.url);
+
+import ofetch from '@/utils/ofetch';
+import { CheerioAPI } from 'cheerio';
+import timezone from '@/utils/timezone';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import path from 'node:path';
+
+const defaultSrc = '_static/ee6af7e.js';
+const defaultToken = 'djflkdsoisknfoklsyhownfrlewfknoiaewf';
+
+const rootUrl = 'https://top.aibase.com';
+const apiRootUrl = 'https://app.chinaz.com';
+
+/**
+ * Converts a string to an array.
+ * If the string starts with '[', it is assumed to be a JSON array and is parsed accordingly.
+ * Otherwise, the string is wrapped in an array.
+ *
+ * @param str - The input string to convert to an array.
+ * @returns An array created from the input string.
+ */
+const strToArray = (str: string) => {
+    if (str.startsWith('[')) {
+        return JSON.parse(str);
+    }
+    return [str];
+};
+
+art.defaults.imports.strToArray = strToArray;
+
+/**
+ * Retrieve a token asynchronously using a CheerioAPI instance.
+ * @param $ - The CheerioAPI instance.
+ * @returns A Promise that resolves to a string representing the token.
+ */
+const getToken = async ($: CheerioAPI): Promise<string> => {
+    const scriptUrl = new URL($('script[src]').last()?.prop('src') ?? defaultSrc, rootUrl).href;
+
+    const script = await ofetch(scriptUrl, {
+        responseType: 'text',
+    });
+
+    return script.match(/"\/(\w+)\/ai\/.*?\.aspx"/)?.[1] ?? defaultToken;
+};
+
+/**
+ * Build API URLs asynchronously using a CheerioAPI instance.
+ * @param $ - The CheerioAPI instance.
+ * @returns An object containing API URLs.
+ */
+const buildApiUrl = async ($: CheerioAPI) => {
+    const token = await getToken($);
+
+    const apiRecommListUrl = new URL(`${token}/ai/GetAIProcRecommList.aspx`, apiRootUrl).href;
+    const apiRecommProcUrl = new URL(`${token}/ai/GetAIProcListByRecomm.aspx`, apiRootUrl).href;
+    const apiTagProcUrl = new URL(`${token}/ai/GetAiProductOfTag.aspx`, apiRootUrl).href;
+    // AI 资讯列表
+    const apiInfoListUrl = new URL(`${token}/ai/GetAiInfoList.aspx`, apiRootUrl).href;
+
+    return {
+        apiRecommListUrl,
+        apiRecommProcUrl,
+        apiTagProcUrl,
+        apiInfoListUrl,
+    };
+};
+
+/**
+ * Process an array of items to generate a new array of processed items for RSS.
+ * @param items - An array of items to process.
+ * @returns An array of processed items.
+ */
+const processItems = (items: any[]): any[] =>
+    items.map((item) => {
+        const title = item.name;
+        const image = item.imgurl;
+        const description = art(path.join(__dirname, 'templates/description.art'), {
+            images: image
+                ? [
+                      {
+                          src: image,
+                          alt: title,
+                      },
+                  ]
+                : undefined,
+            item,
+        });
+        const guid = `aibase-${item.zurl}`;
+
+        return {
+            title,
+            description,
+            pubDate: timezone(parseDate(item.addtime), +8),
+            link: new URL(`tool/${item.zurl}`, rootUrl).href,
+            category: [...new Set([...strToArray(item.categories), ...strToArray(item.tags), item.catname, item.procattrname, item.procformname, item.proctypename])].filter(Boolean),
+            guid,
+            id: guid,
+            content: {
+                html: description,
+                text: item.desc,
+            },
+            image,
+            banner: image,
+            updated: parseDate(item.UpdTime),
+            enclosure_url: item.logo,
+            enclosure_type: item.logo ? `image/${item.logo.split(/\./).pop()}` : undefined,
+            enclosure_title: title,
+        };
+    });
+
+export { rootUrl, processItems, buildApiUrl };

package/lib/routes/anime1/anime.ts

@@ -0,0 +1,63 @@
+import { Route } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+
+export const route: Route = {
+    path: 'anime/:category/:name',
+    name: 'Anime',
+    url: 'anime1.me',
+    maintainers: ['cxheng315'],
+    example: '/anime1/anime/2024年夏季/神之塔-第二季',
+    categories: ['anime'],
+    parameters: {
+        category: 'Anime1 Category',
+        name: 'Anime1 Name',
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['anime1.me/category/:category/:name'],
+            target: '/anime/:category/:name',
+        },
+    ],
+    handler,
+};
+
+async function handler(ctx) {
+    const { category, name } = ctx.req.param();
+
+    const response = await ofetch(`https://anime1.me/category/${category}/${name}`);
+
+    const $ = load(response);
+
+    const title = $('page-title').text().trim();
+
+    const items = $('article')
+        .toArray()
+        .map((el) => {
+            const $el = $(el);
+            const title = $el.find('.entry-title a').text().trim();
+            return {
+                title,
+                link: $el.find('.entry-title a').attr('href'),
+                description: title,
+                pubDate: parseDate($el.find('time').attr('datetime') || ''),
+                itunes_item_image: $el.find('video').attr('poster'),
+            };
+        });
+
+    return {
+        title,
+        link: `https://anime1.me/category/${category}/${name}`,
+        description: title,
+        item: items,
+    };
+}

package/lib/routes/anime1/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Anime1',
+    url: 'anime1.me',
+};

package/lib/routes/anime1/search.ts

@@ -0,0 +1,57 @@
+import { Route } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+
+export const route: Route = {
+    path: 'search/:keyword',
+    name: 'Search',
+    url: 'anime1.me',
+    maintainers: ['cxheng315'],
+    example: '/anime1/search/神之塔',
+    categories: ['anime'],
+    parameters: {
+        keyword: 'Anime1 Search Keyword',
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    handler,
+};
+
+async function handler(ctx) {
+    const { keyword } = ctx.req.param();
+
+    const response = await ofetch(`https://anime1.me/?s=${keyword}`);
+
+    const $ = load(response);
+
+    const title = $('page-title').text().trim();
+
+    const items = $('article.type-post')
+        .toArray()
+        .map((el) => {
+            const $el = $(el);
+            const title = $el.find('.entry-title a').text().trim();
+            return {
+                title,
+                link: $el.find('.entry-title a').attr('href'),
+                description: title,
+                pubDate: parseDate($el.find('time').attr('datetime') || ''),
+            };
+        });
+
+    return {
+        title,
+        link: `https://anime1.me/?s=${keyword}`,
+        description: title,
+        itunes_author: 'Anime1',
+        itunes_image: 'https://anime1.me/wp-content/uploads/2021/02/cropped-1-180x180.png',
+        item: items,
+    };
+}

package/lib/routes/apnews/api.ts

@@ -0,0 +1,77 @@
+import { Route, ViewType } from '@/types';
+import { fetchArticle } from './utils';
+import ofetch from '@/utils/ofetch';
+import timezone from '@/utils/timezone';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/api/:tags?',
+    categories: ['traditional-media', 'popular'],
+    example: '/apnews/api/apf-topnews',
+    view: ViewType.Articles,
+    parameters: {
+        tags: {
+            description: 'Getting a list of articles from a public API based on tags.',
+            options: [
+                { value: 'apf-topnews', label: 'Top News' },
+                { value: 'apf-sports', label: 'Sports' },
+                { value: 'apf-politics', label: 'Politics' },
+                { value: 'apf-entertainment', label: 'Entertainment' },
+                { value: 'apf-usnews', label: 'US News' },
+                { value: 'apf-oddities', label: 'Oddities' },
+                { value: 'apf-Travel', label: 'Travel' },
+                { value: 'apf-technology', label: 'Technology' },
+                { value: 'apf-lifestyle', label: 'Lifestyle' },
+                { value: 'apf-business', label: 'Business' },
+                { value: 'apf-Health', label: 'Health' },
+                { value: 'apf-science', label: 'Science' },
+                { value: 'apf-intlnews', label: 'International News' },
+            ],
+            default: 'apf-topnews',
+        },
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['apnews.com/'],
+        },
+    ],
+    name: 'News',
+    maintainers: ['dzx-dzx'],
+    handler,
+};
+
+async function handler(ctx) {
+    const { tags = 'apf-topnews' } = ctx.req.param();
+    const apiRootUrl = 'https://afs-prod.appspot.com/api/v2/feed/tag';
+    const url = `${apiRootUrl}?tags=${tags}`;
+    const res = await ofetch(url);
+
+    const list = res.cards
+        .map((e) => ({
+            title: e.contents[0]?.headline,
+            link: e.contents[0]?.localLinkUrl,
+            pubDate: timezone(parseDate(e.publishedDate), 0),
+            category: e.tagObjs.map((tag) => tag.name),
+            updated: timezone(parseDate(e.contents[0]?.updated), 0),
+            description: e.contents[0]?.storyHTML,
+            author: e.contents[0]?.reporters.map((author) => ({ name: author.displayName })),
+        }))
+        .sort((a, b) => b.pubDate - a.pubDate)
+        .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20);
+
+    const items = ctx.req.query('mode') === 'fulltext' ? await Promise.all(list.map((item) => fetchArticle(item))) : list;
+
+    return {
+        title: `${res.tagObjs[0].name} - AP News`,
+        item: items,
+        link: 'https://apnews.com',
+    };
+}

package/lib/routes/apnews/rss.ts

@@ -5,7 +5,7 @@ const HOME_PAGE = 'https://apnews.com';
 
 export const route: Route = {
     path: '/rss/:category?',
-    categories: ['traditional-media', 'popular'],
+    categories: ['traditional-media'],
     example: '/apnews/rss/business',
     view: ViewType.Articles,
     parameters: {
@@ -38,7 +38,7 @@ async function handler(ctx) {
     const url = `${HOME_PAGE}/${rss}.rss`;
     const res = await parser.parseURL(url);
 
-    const items = await Promise.all(res.items.map((item) => fetchArticle(item)));
+    const items = ctx.req.query('mode') === 'fulltext' ? await Promise.all(res.items.map((item) => fetchArticle(item))) : res;
 
     return {
         ...res,

package/lib/routes/apnews/sitemap.ts

@@ -0,0 +1,99 @@
+import { Route, ViewType } from '@/types';
+import { fetchArticle } from './utils';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import asyncPool from 'tiny-async-pool';
+import timezone from '@/utils/timezone';
+const HOME_PAGE = 'https://apnews.com';
+
+export const route: Route = {
+    path: '/sitemap/:route',
+    categories: ['traditional-media'],
+    example: '/apnews/sitemap/ap-sitemap-latest',
+    view: ViewType.Articles,
+    parameters: {
+        route: {
+            description: 'Route for sitemap, excluding the `.xml` extension',
+        },
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['apnews.com/'],
+        },
+    ],
+    name: 'Sitemap',
+    maintainers: ['zoenglinghou', 'mjysci', 'TonyRL', 'dzx-dzx'],
+    handler,
+};
+
+async function handler(ctx) {
+    const route = ctx.req.param('route');
+    const url = `${HOME_PAGE}/${route}.xml`;
+    const response = await ofetch(url);
+    const $ = load(response);
+
+    const list = $('urlset url')
+        .toArray()
+        .map((e) => {
+            const LANGUAGE_MAP = new Map([
+                ['eng', 'en'],
+                ['spa', 'es'],
+            ]);
+
+            const title = $(e)
+                .find(String.raw`news\:title`)
+                .text();
+            const pubDate = parseDate(
+                $(e)
+                    .find(String.raw`news\:publication_date`)
+                    .text()
+            );
+            const lastmod = timezone(parseDate($(e).find(`lastmod`).text()), -4);
+            const language = LANGUAGE_MAP.get(
+                $(e)
+                    .find(String.raw`news\:language`)
+                    .text()
+            );
+            let res = { link: $(e).find('loc').text() };
+            if (title) {
+                res = Object.assign(res, { title });
+            }
+            if (pubDate.toString() !== 'Invalid Date') {
+                res = Object.assign(res, { pubDate });
+            }
+            if (language) {
+                res = Object.assign(res, { language });
+            }
+            if (lastmod.toString() !== 'Invalid Date') {
+                res = Object.assign(res, { lastmod });
+            }
+            return res;
+        })
+        .filter((e) => Boolean(e.link) && !new URL(e.link).pathname.split('/').includes('hub'))
+        .sort((a, b) => (a.pubDate && b.pubDate ? b.pubDate - a.pubDate : b.lastmod - a.lastmod))
+        .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20);
+
+    const items = ctx.req.query('mode') === 'fulltext' ? await asyncPoolAll(20, list, (item) => fetchArticle(item)) : list;
+
+    return {
+        title: `AP News sitemap:${route}`,
+        item: items,
+        link: 'https://apnews.com',
+    };
+}
+async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
+    const results: Awaited<OUT[]> = [];
+    for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
+        results.push(result);
+    }
+    return results;
+}

package/lib/routes/apnews/topics.ts

@@ -42,14 +42,14 @@ async function handler(ctx) {
 
     const items = await Promise.all(
         $(':is(.PagePromo-content, .PageListStandardE-leadPromo-info) bsp-custom-headline')
-            .get()
+            .toArray()
             .slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : Infinity)
             .map((e) => ({
                 title: $(e).find('span.PagePromoContentIcons-text').text(),
                 link: $(e).find('a').attr('href'),
             }))
             .filter((e) => typeof e.link === 'string')
-            .map((item) => fetchArticle(item))
+            .map((item) => (ctx.req.query('mode') === 'fulltext' ? fetchArticle(item) : item))
     );
 
     return {

package/lib/routes/apnews/utils.ts

@@ -17,12 +17,13 @@ export function fetchArticle(item) {
         const rawLdjson = JSON.parse($('#link-ld-json').text());
         let ldjson;
         if (rawLdjson['@type'] === 'NewsArticle' || (Array.isArray(rawLdjson) && rawLdjson.some((e) => e['@type'] === 'NewsArticle'))) {
-            // Regular
+            // Regular(Articles, Videos)
             ldjson = Array.isArray(rawLdjson) ? rawLdjson.find((e) => e['@type'] === 'NewsArticle') : rawLdjson;
 
             $('div.Enhancement').remove();
             const section = $("meta[property='article:section']").attr('content');
             return {
+                title: ldjson.headline,
                 pubDate: parseDate(ldjson.datePublished),
                 updated: parseDate(ldjson.dateModified),
                 description: $('div.RichTextStoryBody').html() || $(':is(.VideoLead, .VideoPage-pageSubHeading)').html(),

package/lib/routes/apple/podcast.ts

@@ -24,7 +24,7 @@ export const route: Route = {
     name: '播客',
     maintainers: ['Acring'],
     handler,
-    url: 'https://www.apple.com.cn/apple-podcasts/',
+    url: 'www.apple.com.cn/apple-podcasts/',
 };
 
 async function handler(ctx) {
@@ -36,29 +36,33 @@ async function handler(ctx) {
 
     const $ = load(response.data);
 
-    const page_data = JSON.parse($('#shoebox-media-api-cache-amp-podcasts').text());
+    const serializedServerData = JSON.parse($('#serialized-server-data').text());
 
-    const data = JSON.parse(page_data[Object.keys(page_data)[0]]).d[0];
-    const attributes = data.attributes;
+    const seoEpisodes = serializedServerData[0].data.seoData.schemaContent.workExample;
+    const originEpisodes = serializedServerData[0].data.shelves.find((item) => item.contentType === 'episode').items;
+    const header = serializedServerData[0].data.shelves.find((item) => item.contentType === 'showHeaderRegular').items[0];
+
+    const episodes = originEpisodes.map((item) => {
+        // Try to keep line breaks in the description
+        const matchedSeoEpisode = seoEpisodes.find((seoEpisode) => seoEpisode.name === item.title) || null;
+        const episodeDescription = (matchedSeoEpisode ? matchedSeoEpisode.description : item.summary).replaceAll('\n', '<br>');
 
-    const episodes = data.relationships.episodes.data.map((item) => {
-        const attr = item.attributes;
         return {
-            title: attr.name,
-            enclosure_url: attr.assetUrl,
-            itunes_duration: attr.durationInMilliseconds / 1000,
+            title: item.title,
+            enclosure_url: item.playAction.episodeOffer.streamUrl,
             enclosure_type: 'audio/mp4',
-            link: attr.url,
-            pubDate: parseDate(attr.releaseDateTime),
-            description: attr.description.standard.replaceAll('\n', '<br>'),
+            itunes_duration: item.duration,
+            link: item.playAction.episodeOffer.storeUrl,
+            pubDate: parseDate(item.releaseDate),
+            description: episodeDescription,
         };
     });
 
     return {
-        title: attributes.name,
-        link: attributes.url,
-        itunes_author: attributes.artistName,
+        title: header.title,
+        link: header.contextAction.podcastOffer.storeUrl,
+        itunes_author: header.contextAction.podcastOffer.author,
         item: episodes,
-        description: attributes.description.standard,
+        description: header.description.replaceAll('\n', ' '),
     };
 }