rsshub 1.0.0-master.ff487f3 → 1.0.0-master.ff76aa7

package/lib/routes/dcfever/utils.ts

@@ -63,11 +63,20 @@ const parseTradeItem = (item) =>
         const response = await ofetch(item.link);
         const $ = load(response);
 
-        $('.selector_text').remove();
-        $('.selector_image_div').each((_, div) => {
+        const photoSelector = $('#trading_item_section .description')
+            .contents()
+            .filter((_, e) => e.type === 'comment')
+            .toArray()
+            .map((e) => e.data)
+            .join('');
+
+        const $photo = load(photoSelector, null, false);
+
+        $photo('.selector_text').remove();
+        $photo('.selector_image_div').each((_, div) => {
             delete div.attribs.onclick;
         });
-        $('.desktop_photo_selector img').each((_, img) => {
+        $photo('.desktop_photo_selector img').each((_, img) => {
             if (img.attribs.src.endsWith('_sqt.jpg')) {
                 img.attribs.src = img.attribs.src.replace('_sqt.jpg', '.jpg');
             }
@@ -76,7 +85,7 @@ const parseTradeItem = (item) =>
         item.description = art(path.join(__dirname, 'templates/trading.art'), {
             info: $('.info_col'),
             description: $('.description_text').html(),
-            photo: $('.desktop_photo_selector').html(),
+            photo: $photo('.desktop_photo_selector').html(),
         });
 
         return item;

package/lib/routes/dealstreetasia/home.ts

@@ -0,0 +1,72 @@
+import { Route } from '@/types';
+// import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch'; // Unified request library used
+import { load } from 'cheerio'; // An HTML parser with an API similar to jQuery
+// import puppeteer from '@/utils/puppeteer';
+// import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/home',
+    categories: ['traditional-media'],
+    example: '/dealstreetasia/home',
+    // parameters: { section: 'target section' },
+    radar: [
+        {
+            source: ['dealstreetasia.com/'],
+        },
+    ],
+    name: 'Home',
+    maintainers: ['jack2game'],
+    handler,
+    url: 'dealstreetasia.com/',
+};
+
+async function handler() {
+    // const section = ctx.req.param('section');
+    const items = await fetchPage();
+
+    return items;
+}
+
+async function fetchPage() {
+    const baseUrl = 'https://dealstreetasia.com'; // Define base URL
+
+    const response = await ofetch(`${baseUrl}/`);
+    const $ = load(response);
+
+    const jsonData = JSON.parse($('#__NEXT_DATA__').text());
+    // const headingText = jsonData.props.pageProps.sectionData.name;
+
+    const pageProps = jsonData.props.pageProps;
+    const list = [
+        ...pageProps.topStories,
+        ...pageProps.privateEquity,
+        ...pageProps.ventureCapital,
+        ...pageProps.unicorns,
+        ...pageProps.interviews,
+        ...pageProps.deals,
+        ...pageProps.analysis,
+        ...pageProps.ipos,
+        ...pageProps.opinion,
+        ...pageProps.policyAndRegulations,
+        ...pageProps.people,
+        ...pageProps.earningsAndResults,
+        ...pageProps.theLpView,
+        ...pageProps.dvNewsletters,
+        ...pageProps.reports,
+    ].map((item) => ({
+        title: item.post_title || item.title || 'No Title',
+        link: item.post_url || item.link || '',
+        description: item.post_excerpt || item.excerpt || '',
+        pubDate: item.post_date ? new Date(item.post_date).toUTCString() : item.date ? new Date(item.date).toUTCString() : '',
+        category: item.category_link ? item.category_link.replaceAll(/(<([^>]+)>)/gi, '') : '', // Clean HTML if category_link exists
+        image: item.image_url ? item.image_url.replace(/\?.*$/, '') : '', // Remove query parameters if image_url exists
+    }));
+
+    return {
+        title: 'Deal Street Asia',
+        language: 'en',
+        item: list,
+        link: 'https://dealstreetasia.com/',
+    };
+}

package/lib/routes/dealstreetasia/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'DealStreetAsia',
+    url: 'dealstreetasia.com',
+};

package/lib/routes/dealstreetasia/section.ts

@@ -0,0 +1,57 @@
+import { Route } from '@/types';
+// import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch'; // Unified request library used
+import { load } from 'cheerio'; // An HTML parser with an API similar to jQuery
+// import puppeteer from '@/utils/puppeteer';
+// import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/section/:section',
+    categories: ['traditional-media'],
+    example: '/dealstreetasia/section/private-equity',
+    parameters: { section: 'target section' },
+    radar: [
+        {
+            source: ['dealstreetasia.com/'],
+        },
+    ],
+    name: 'Section',
+    maintainers: ['jack2game'],
+    handler,
+    url: 'dealstreetasia.com/',
+};
+
+async function handler(ctx) {
+    const section = ctx.req.param('section');
+    const items = await fetchPage(section);
+
+    return items;
+}
+
+async function fetchPage(section: string) {
+    const baseUrl = 'https://dealstreetasia.com'; // Define base URL
+
+    const response = await ofetch(`${baseUrl}/section/${section}/`);
+    const $ = load(response);
+
+    const jsonData = JSON.parse($('#__NEXT_DATA__').text());
+    const headingText = jsonData.props.pageProps.sectionData.name;
+
+    const items = jsonData.props.pageProps.sectionData.stories.nodes;
+
+    const feedItems = items.map((item) => ({
+        title: item.title || 'No Title',
+        link: item.uri ? `https://www.dealstreetasia.com${item.uri}` : '',
+        description: item.excerpt || '', // Default to empty string if undefined
+        pubDate: item.post_date ? new Date(item.post_date).toUTCString() : '',
+        category: item.sections.nodes.map((section) => section.name),
+        image: item.featuredImage?.node?.mediaItemUrl.replace(/\?.*$/, ''), // Use .replace to sanitize the image URL
+    }));
+
+    return {
+        title: 'Deal Street Asia - ' + headingText,
+        language: 'en',
+        item: feedItems,
+        link: 'https://dealstreetasia.com/section/' + section + '/',
+    };
+}

package/lib/routes/deepin/thread.ts

@@ -0,0 +1,102 @@
+import { Route, DataItem } from '@/types';
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/threads/:type?',
+    categories: ['bbs'],
+    example: '/deepin/threads/latest',
+    parameters: {
+        type: {
+            description: '主题类型',
+            options: [
+                {
+                    value: 'hot',
+                    label: '最热主题',
+                },
+                {
+                    value: 'latest',
+                    label: '最新主题',
+                },
+            ],
+        },
+    },
+    name: '首页主题列表',
+    maintainers: ['myml'],
+    radar: [
+        {
+            source: ['bbs.deepin.org'],
+            target: '/threads/latest',
+        },
+    ],
+    handler,
+};
+
+interface ThreadIndexResult {
+    ThreadIndex: {
+        id: number;
+        subject: string;
+        created_at: string;
+        user: { nickname: string };
+        forum: { name: string };
+    }[];
+}
+interface ThreadInfoResult {
+    data: {
+        id: number;
+        subject: string;
+        created_at: string;
+        user: { nickname: string };
+        post: { message: string };
+    };
+}
+
+const TypeMap = {
+    hot: { where: 'hot_value', title: '最热主题' },
+    latest: { where: 'id', title: '最新主题' },
+};
+
+async function handler(ctx) {
+    let type = TypeMap.latest;
+    if (ctx.req.param('type') === 'hot') {
+        type = TypeMap.hot;
+    }
+    const res = await ofetch<ThreadIndexResult>('https://bbs.deepin.org.cn/api/v1/thread/index', {
+        query: {
+            languages: 'zh_CN',
+            order: 'updated_at',
+            where: type.where,
+        },
+        headers: {
+            accept: 'application/json',
+        },
+    });
+    const items = await Promise.all(
+        res.ThreadIndex.map((thread) => {
+            const link = 'https://bbs.deepin.org.cn/post/' + thread.id;
+            return cache.tryGet(link, async () => {
+                const item: DataItem = {
+                    id: String(thread.id),
+                    title: thread.subject,
+                    link,
+                    pubDate: parseDate(thread.created_at),
+                    author: thread.user.nickname,
+                    category: [thread.forum.name],
+                    description: '',
+                };
+                const cacheData = await ofetch<ThreadInfoResult>('https://bbs.deepin.org.cn/api/v1/thread/info?id=' + item.id);
+                if (cacheData) {
+                    const info = cacheData as ThreadInfoResult;
+                    item.description = info.data.post.message;
+                }
+                return item;
+            }) as Promise<DataItem>;
+        })
+    );
+    return {
+        title: 'deepin论坛主页 - ' + type.title,
+        link: 'https://bbs.deepin.org',
+        item: items,
+    };
+}

package/lib/routes/deeplearning/thebatch.ts

@@ -1,6 +1,8 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
+import * as cheerio from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/thebatch',
@@ -27,33 +29,43 @@ export const route: Route = {
 };
 
 async function handler() {
-    const page = await got({
-        method: 'get',
-        url: `https://www.deeplearning.ai/the-batch/`,
-    });
-    const nextJs = page.data.match(/<script id="__NEXT_DATA__" type="application\/json">(.*)<\/script>/)[1];
+    const baseUrl = 'https://www.deeplearning.ai';
+    const link = `${baseUrl}/the-batch/`;
+    const page = await ofetch(link);
+    const $ = cheerio.load(page);
+    const nextJs = $('script#__NEXT_DATA__').text();
     const nextBuildId = JSON.parse(nextJs).buildId;
 
-    const listing = await got({
-        method: 'get',
-        url: `https://www.deeplearning.ai/_next/data/${nextBuildId}/the-batch.json`,
-    });
+    const listing = await ofetch(`${baseUrl}/_next/data/${nextBuildId}/the-batch.json`);
 
-    const items = listing.data.pageProps.posts.map((item) => ({
+    const items = listing.pageProps.posts.map((item) => ({
         title: item.title,
-        link: `https://www.deeplearning.ai/the-batch/${item.slug}`,
-        jsonUrl: `https://www.deeplearning.ai/_next/data/${nextBuildId}/the-batch/${item.slug}.json`,
-        pubDate: new Date(item.published_at).toUTCString(),
+        link: `${link}${item.slug}`,
+        jsonUrl: `${baseUrl}/_next/data/${nextBuildId}/the-batch/${item.slug}.json`,
+        pubDate: parseDate(item.published_at),
     }));
 
     return {
-        title: `The Batch - a new weekly newsletter from deeplearning.ai`,
-        link: `https://www.deeplearning.ai/the-batch/`,
+        title: 'The Batch - a new weekly newsletter from deeplearning.ai',
+        link,
         item: await Promise.all(
             items.map((item) =>
                 cache.tryGet(item.link, async () => {
-                    const resp = await got({ method: 'get', url: item.jsonUrl });
-                    item.description = resp.data.pageProps.cmsData.post.html;
+                    const resp = await ofetch(item.jsonUrl);
+                    const $ = cheerio.load(resp.pageProps.cmsData.post.html);
+
+                    $('a').each((_, ele) => {
+                        if (ele.attribs.href?.includes('utm_campaign')) {
+                            const url = new URL(ele.attribs.href);
+                            url.searchParams.delete('utm_campaign');
+                            url.searchParams.delete('utm_source');
+                            url.searchParams.delete('utm_medium');
+                            url.searchParams.delete('_hsenc');
+                            ele.attribs.href = url.href;
+                        }
+                    });
+
+                    item.description = $.html();
                     return item;
                 })
             )

package/lib/routes/devolverdigital/blog.ts

@@ -1,11 +1,10 @@
-import { Route } from '@/types';
-import cache from '@/utils/cache';
-import got from '@/utils/got';
+import { DataItem, Route } from '@/types';
 import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
 
 export const route: Route = {
     path: '/blog',
-    categories: ['blog'],
+    categories: ['game'],
     example: '/devolverdigital/blog',
     parameters: {},
     features: {
@@ -28,44 +27,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const baseUrl = 'https://www.devolverdigital.com/blog';
-    const { data: response } = await got(baseUrl);
-
-    const $ = load(response);
-    const nextData = JSON.parse($('#__NEXT_DATA__').text());
-
-    const items = await Promise.all(
-        nextData.props.pageProps.posts.map((postData) => {
-            const postUrl = `${baseUrl}/post/${postData.id}`;
-            return cache.tryGet(postUrl, async () => {
-                const { data: postPage } = await got(postUrl);
-
-                const $page = load(postPage);
-                $page('noscript').remove();
-                const postContent = $page('div.flex > div > div > div > div:not([class])');
-
-                // img resource redirection and
-                // clean up absolute layouts for img and span
-                const imageUrls = postData.body.filter((item) => item.type === 'upload' && item.value.cloudinary.resource_type === 'image').map((item) => item.value.cloudinary.secure_url);
-                const allImageSpans = postContent.find('span > img').parent();
-                allImageSpans.each((spanIndex, span) => {
-                    $(span).attr('style', $(span).attr('style').replace('position:absolute', ''));
-                    const img = $(span).find('img');
-                    img.attr('src', imageUrls[spanIndex]);
-                    img.attr('style', img.attr('style').replace('position:absolute', '').replace('width:0', '').replace('height:0', ''));
-                });
-
-                return {
-                    title: postData.title,
-                    link: postUrl,
-                    author: postData.author,
-                    pubDate: Date.parse(postData.createdAt),
-                    updated: Date.parse(postData.updatedAt),
-                    description: postContent.html(),
-                };
-            });
-        })
-    );
+    const items = await fetchPage();
 
     return {
         title: 'DevolverDigital Blog',
@@ -74,3 +36,58 @@ async function handler() {
         item: items,
     };
 }
+
+async function fetchPage() {
+    const baseUrl = 'https://www.devolverdigital.com/blog';
+    const response = await ofetch(baseUrl);
+    const $ = load(response, { scriptingEnabled: false });
+
+    // Extract all posts of this page
+    const $titleDivs = $('div.w-full.flex.justify-center.py-4.bg-red-400.undefined');
+    const $contentDivs = $('div.bg-gray-800.flex.justify-center.font-sm.py-4');
+    const items: DataItem[] = $titleDivs.toArray().map((titleDiv, index) => {
+        const content = $contentDivs[index];
+        const postAuthor = parsePostAuthor($, titleDiv);
+        const postDate = parsePostDate($, titleDiv);
+        const postTitle = $(titleDiv).find('h1').text();
+        const postLink = $(content).find('div.ml-auto.flex.items-center a').attr('href');
+        // Modify the src attribute of the image
+        parsePostImages($, content);
+        const postContent = $.html($(content).find('div.cms-content'));
+        return {
+            title: postTitle,
+            link: postLink,
+            author: postAuthor,
+            pubDate: postDate,
+            description: postContent,
+        };
+    });
+
+    return items;
+}
+
+function parsePostAuthor($, titleDiv) {
+    const $postAuthorElement = $(titleDiv).find('div.font-xs.leading-none.mb-1');
+    return $postAuthorElement.text().replace('By ', '') || 'Devolver Digital';
+}
+
+function parsePostDate($, titleDiv) {
+    const dateStr = $(titleDiv).find('div.font-2xs.leading-none.mb-1').text();
+    const cleanedDateStr = dateStr.replace(/(\d+)(st|nd|rd|th)/, '$1');
+    return new Date(cleanedDateStr);
+}
+
+function parsePostImages($, content) {
+    $(content)
+        .find('img')
+        .each((index, img) => {
+            const $img = $(img);
+            const src = $img.attr('src') || '';
+            if (src.startsWith('/_next/image')) {
+                const srcSet = $img.attr('srcset') || '';
+                const actualSrc = srcSet.split(',').pop()?.split(' ')[0] || src;
+                $img.attr('src', actualSrc);
+            }
+            $img.removeAttr('loading').removeAttr('decoding').removeAttr('data-nimg').removeAttr('style').removeAttr('sizes').removeAttr('srcset').removeAttr('referrerpolicy');
+        });
+}

package/lib/routes/devtrium/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Devtrium',
+    url: 'devtrium.com',
+};

package/lib/routes/devtrium/posts.ts

@@ -0,0 +1,55 @@
+import { DataItem, Route } from '@/types';
+import { load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/',
+    categories: ['programming'],
+    example: '/devtrium',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['devtrium.com'],
+        },
+    ],
+    name: 'Official Blogs',
+    maintainers: ['Xy2002'],
+    handler,
+    url: 'devtrium.com',
+};
+
+async function handler() {
+    const items = await fetchPage();
+
+    return {
+        title: 'Devtrium',
+        language: 'en-us',
+        item: items,
+        link: 'https://devtrium.com',
+    };
+}
+
+async function fetchPage() {
+    const baseUrl = 'https://devtrium.com';
+    const response = await ofetch(baseUrl);
+    const $ = load(response, { scriptingEnabled: false });
+
+    // Extract all posts of this page
+    const data = JSON.parse($('script#__NEXT_DATA__').text());
+    const items: DataItem[] = data.props.pageProps.posts.map((post) => ({
+        title: post.title,
+        link: `${baseUrl}/posts/${post.slug}`,
+        description: post.description,
+        pubDate: parseDate(post.date),
+    }));
+    return items;
+}

package/lib/routes/discourse/official.ts

@@ -0,0 +1,55 @@
+import { Route } from '@/types';
+import { getConfig } from './utils';
+import got from '@/utils/got';
+import RSSParser from '@/utils/rss-parser';
+
+export const route: Route = {
+    path: '/:configId/official/:path{.+}',
+    categories: ['bbs'],
+    example: '/discourse/0/official/latest',
+    parameters: {
+        configId: 'Environment variable configuration id, see above',
+        path: 'Discourse RSS path between `domain` and `.rss`. All supported Rss path can be found in [https://meta.discourse.org/t/finding-discourse-rss-feeds/264134](https://meta.discourse.org/t/finding-discourse-rss-feeds/264134). For example: the path of [https://meta.discourse.org/top/all.rss](https://meta.discourse.org/top/all.rss) is `top/all`.',
+    },
+    features: {
+        requireConfig: [
+            {
+                name: 'DISCOURSE_CONFIG_*',
+                description: `Configure the Discourse environment variables referring to [https://docs.rsshub.app/deploy/config#discourse](https://docs.rsshub.app/deploy/config#discourse).`,
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Official RSS',
+    maintainers: ['Raikyou', 'dzx-dzx'],
+    handler,
+};
+
+async function handler(ctx) {
+    const { link, key } = getConfig(ctx);
+    const path = ctx.req.param('path');
+
+    const url = `${link}/${path}.rss`;
+
+    const feed = await RSSParser.parseString(
+        (
+            await got(url, {
+                headers: {
+                    'User-Api-Key': key,
+                },
+            })
+        ).data
+    );
+
+    feed.items = feed.items.map((e) => ({
+        description: e.content,
+        author: e.creator,
+        ...e,
+    }));
+
+    return { item: feed.items, ...feed };
+}

package/lib/routes/dlsite/campaign.ts

@@ -149,9 +149,8 @@ async function handler(ctx) {
     }
     const link = setUrl(info);
 
-    const response = await got(link, {
+    const response = await got(new URL(link, host), {
         method: 'GET',
-        prefixUrl: host,
     });
     const data = response.data;
     const $ = load(data);

package/lib/routes/dlsite/new.ts

@@ -81,9 +81,8 @@ async function handler(ctx) {
 
     const link = info.url.slice(1);
 
-    const response = await got(link, {
+    const response = await got(new URL(link, host), {
         method: 'GET',
-        prefixUrl: host,
     });
     const data = response.data;
     const $ = load(data);

package/lib/routes/dlsite/{index.ts → z-index/index.ts}

@@ -1,5 +1,5 @@
 import { Route } from '@/types';
-import { ProcessItems } from './utils';
+import { ProcessItems } from '../utils';
 
 export const route: Route = {
     path: '*',

package/lib/routes/dockerhub/build.ts

@@ -6,7 +6,7 @@ export const route: Route = {
     path: '/build/:owner/:image/:tag?',
     categories: ['program-update', 'popular'],
     view: ViewType.Notifications,
-    example: '/dockerhub/build/diygod/rsshub',
+    example: '/dockerhub/build/diygod/rsshub/latest',
     parameters: {
         owner: 'Image owner, the owner of the official image fills in the library, for example: /dockerhub/build/library/mysql',
         image: 'Image name',

package/lib/routes/dockerhub/utils.ts

@@ -2,9 +2,10 @@ import md5 from '@/utils/md5';
 
 function hash(images) {
     const entries = Object.entries(images)
-        .map((x) => [`${x[1].os}/${x[1].architecture}`, x[1].digest])
-        .sort((a, b) => a[0] - b[0]);
-    return md5(entries.map((x) => x.join(',')).join('|'));
+        .map((x) => `${x[1].os}/${x[1].architecture},${x[1].digest}`)
+        .sort((a, b) => a.localeCompare(b));
+    const text = entries.join('|');
+    return md5(text);
 }
 
 export { hash };