rsshub 1.0.0-master.f72af1b → 1.0.0-master.f7347d9

package/lib/routes/douyin/user.ts

@@ -4,8 +4,11 @@ import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
 import { config } from '@/config';
 import { fallback, queryToBoolean } from '@/utils/readable-social';
-import { templates, resolveUrl, proxyVideo, getOriginAvatar, universalGet } from './utils';
+import { templates, resolveUrl, proxyVideo, getOriginAvatar } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
+import puppeteer from '@/utils/puppeteer';
+import logger from '@/utils/logger';
+import { PostData } from './types';
 
 export const route: Route = {
     path: '/user/:uid/:routeParams?',
@@ -43,29 +46,56 @@ async function handler(ctx) {
 
     const pageUrl = `https://www.douyin.com/user/${uid}`;
 
-    const pageData = await cache.tryGet(
+    const pageData = (await cache.tryGet(
         `douyin:user:${uid}`,
         async () => {
-            const renderData = await universalGet(pageUrl, 'user');
-            const dataKey = Object.keys(renderData).find((key) => renderData[key].user && renderData[key].post);
-            return renderData[dataKey];
+            let postData;
+            const browser = await puppeteer();
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+            });
+            page.on('response', async (response) => {
+                const request = response.request();
+                if (request.url().includes('/web/aweme/post') && !postData) {
+                    postData = await response.json();
+                }
+            });
+
+            logger.http(`Requesting ${pageUrl}`);
+            await page.goto(pageUrl, {
+                waitUntil: 'networkidle2',
+            });
+
+            browser.close();
+
+            if (!postData) {
+                throw new Error('Empty post data. The request may be filtered by WAF.');
+            }
+
+            return postData;
         },
         config.cache.routeExpire,
         false
-    );
-    const userInfo = pageData.user.user;
+    )) as PostData;
+
+    if (!pageData.aweme_list?.length) {
+        throw new Error('Empty post data. The request may be filtered by WAF.');
+    }
+    const userInfo = pageData.aweme_list[0].author;
     const userNickName = userInfo.nickname;
-    const userDescription = userInfo.desc;
-    const userAvatar = getOriginAvatar(userInfo.avatar300Url || userInfo.avatarUrl);
+    // const userDescription = userInfo.desc;
+    const userAvatar = getOriginAvatar(userInfo.avatar_thumb.url_list.at(-1));
 
-    const posts = pageData.post.data;
-    const items = posts.map((post) => {
+    const items = pageData.aweme_list.map((post) => {
         // parse video
-        let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
+        let videoList = post.video?.bit_rate?.map((item) => resolveUrl(item.play_addr.url_list.at(-1)));
         if (relay) {
             videoList = videoList.map((item) => proxyVideo(item, relay));
         }
-        let duration = post.video && post.video.duration;
+        let duration = post.video?.duration;
         duration = duration && duration / 1000;
         let img;
         // if (!embed) {
@@ -73,30 +103,28 @@ async function handler(ctx) {
         // }
         img =
             img ||
-            (post.video &&
-                ((post.video.coverUrlList && post.video.coverUrlList.at(-1)) || // HD
-                    post.video.originCover || // LD
-                    post.video.cover)); // even more LD
+            post.video?.cover?.url_list.at(-1) || // HD
+            post.video?.origin_cover?.url_list.at(-1); // LD
         img = img && resolveUrl(img);
 
         // render description
-        const desc = post.desc && post.desc.replaceAll('\n', '<br>');
+        const desc = post.desc?.replaceAll('\n', '<br>');
         let media = art(embed && videoList ? templates.embed : templates.cover, { img, videoList, duration });
         media = embed && videoList && iframe ? art(templates.iframe, { content: media }) : media; // warp in iframe
         const description = art(templates.desc, { desc, media });
 
         return {
-            title: post.desc,
+            title: post.desc.split('\n')[0],
             description,
-            link: `https://www.douyin.com/video/${post.awemeId}`,
-            pubDate: parseDate(post.createTime * 1000),
-            category: post.textExtra.map((extra) => extra.hashtagName),
+            link: `https://www.douyin.com/video/${post.aweme_id}`,
+            pubDate: parseDate(post.create_time * 1000),
+            category: post.video_tag.map((t) => t.tag_name),
         };
     });
 
     return {
         title: userNickName,
-        description: userDescription,
+        // description: userDescription,
         image: userAvatar,
         link: pageUrl,
         item: items,

package/lib/routes/douyin/utils.ts

@@ -1,13 +1,7 @@
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
-import cache from '@/utils/cache';
 import path from 'node:path';
-import got from '@/utils/got';
-import { load } from 'cheerio';
-import logger from '@/utils/logger';
-import { config } from '@/config';
-import puppeteer from '@/utils/puppeteer';
 
 const templates = {
     desc: path.join(__dirname, 'templates/desc.art'),
@@ -51,84 +45,4 @@ const getOriginAvatar = (url) =>
         .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
         .replaceAll(/~\w+_\d+x\d+/g, '');
 
-const extractRenderData = (html) => {
-    if (!html) {
-        throw new Error('Empty html. The request may be filtered by WAF.');
-    }
-
-    const $ = load(html);
-    const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
-    if (!renderData) {
-        const title = $('title').text();
-        if (title.includes('验证')) {
-            throw new Error(title);
-        } else if ($('#captcha_container').length > 0) {
-            throw new Error('Captcha required.');
-        }
-        throw new Error('Failed to get render data.');
-    } else if (renderData.isSpider) {
-        throw new Error('The request was considered to be from a spider.');
-    } else {
-        return renderData;
-    }
-};
-
-const gotGet = async (url, ua) => {
-    const response = await got(url, {
-        headers: {
-            'User-Agent': ua, // magic!
-        },
-    });
-    return extractRenderData(response.data);
-};
-
-const puppeteerGet = async (pageUrl) => {
-    const browser = await puppeteer();
-    const page = await browser.newPage();
-    await page.setRequestInterception(true);
-    page.on('request', (request) => {
-        request.resourceType() === 'document' ? request.continue() : request.abort();
-    });
-    await page.goto(pageUrl, {
-        waitUntil: 'domcontentloaded',
-    });
-    const html = await page.evaluate(() => document.documentElement.innerHTML);
-    await browser.close();
-    return extractRenderData(html);
-};
-
-const universalGet = async (url, route) => {
-    const cacheKey = `douyin:utils:universalGet:gotFirst:${route}`;
-    const gotFirst = await cache.tryGet(
-        cacheKey,
-        () => '1',
-        config.cache.contentExpire,
-        false // no refresh now
-    );
-
-    const ua = route === 'live' ? 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' : 'curl/7.86.0';
-
-    let data;
-    if (gotFirst === '1') {
-        try {
-            data = await gotGet(url, ua);
-            cache.set(cacheKey, '1', config.cache.contentExpire); // refresh if success
-        } catch (error) {
-            logger.warn('Failed to get data with got, trying puppeteer: ' + error.message);
-            data = await puppeteerGet(url);
-            cache.set(cacheKey, '0', config.cache.contentExpire); // only set if success
-        }
-    } else {
-        try {
-            data = await puppeteerGet(url);
-            cache.set(cacheKey, '0', config.cache.contentExpire); // refresh if success
-        } catch (error) {
-            logger.warn('Failed to get data with puppeteer, trying got: ' + error.message);
-            data = await gotGet(url, ua);
-            cache.set(cacheKey, '1', config.cache.contentExpire); // only set if success
-        }
-    }
-    return data;
-};
-
-export { templates, resolveUrl, proxyVideo, getOriginAvatar, universalGet };
+export { templates, resolveUrl, proxyVideo, getOriginAvatar };

package/lib/routes/dribbble/keyword.ts

@@ -25,5 +25,5 @@ async function handler(ctx) {
 
     const title = `Dribbble - keyword ${keyword}`;
 
-    return await utils.getData(ctx, url, title);
+    return await utils.getData(url, title);
 }

package/lib/routes/dribbble/popular.ts

@@ -32,5 +32,5 @@ async function handler(ctx) {
 
     const title = 'Dribbble - Popular Shots';
 
-    return await utils.getData(ctx, url, title);
+    return await utils.getData(url, title);
 }

package/lib/routes/dribbble/user.ts

@@ -30,5 +30,5 @@ async function handler(ctx) {
 
     const title = `Dribbble - user ${name}`;
 
-    return await utils.getData(ctx, url, title);
+    return await utils.getData(url, title);
 }

package/lib/routes/dribbble/utils.ts

@@ -2,22 +2,24 @@ import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
 import path from 'node:path';
 
+const host = 'https://dribbble.com';
+
 // Refactored function to load a link asynchronously
 async function loadContent(link) {
     // Make a GET request to the specified  and retrieve the response
-    const response = await got(link);
-    const $ = load(response.data);
+    const response = await ofetch(link);
+    const $ = load(response);
 
     const shotData = JSON.parse(
         $('script')
             .text()
-            .match(/shotData:\s({.+?}),\n/)[1]
+            .match(/shotData:\s({.+?}),\n/)?.[1] ?? '{}'
     );
 
     // Join multiple shots together by selecting elements with class 'media-shot' or 'main-shot' or 'block-media-wrapper'
@@ -91,9 +93,7 @@ async function loadContent(link) {
 
 // Refactored code with comments for clarity
 
-function ProcessFeed(list, caches) {
-    const host = 'https://dribbble.com';
-
+function ProcessFeed(list) {
     // Use Promise.all to process all items in the list asynchronously
     return Promise.all(
         list.map((item) => {
@@ -101,18 +101,20 @@ function ProcessFeed(list, caches) {
 
             // The link of item is "/signup/new" when access "https://dribbble.com/search/something"
             // So get url by id
-            const itemId = $(item).attr('id').replace('screenshot-', '');
+            const itemId = $(item).data('thumbnail-id');
 
             // Construct the full item URL using the host and the item ID
-            const itemUrl = new URL(`/shots/${itemId}`, host).href;
+            const guid = new URL(`/shots/${itemId}`, host).href;
+            const itemUrl = new URL($(item).find('.shot-thumbnail-link').attr('href')!, host).href;
 
             // Return a Promise that resolves to an object combining the single item data and the additional data
-            return caches.tryGet(itemUrl, async () => {
+            return cache.tryGet(guid, async () => {
                 const { description, pubDate, author, category } = await loadContent(itemUrl);
 
                 return {
                     title: $('.shot-title').text(),
                     link: itemUrl,
+                    guid,
                     description,
                     pubDate,
                     author,
@@ -126,25 +128,21 @@ function ProcessFeed(list, caches) {
 /**
  * Retrieves data from a given URL and processes it.
  *
- * @param {Object} ctx - The context object.
  * @param {string} url - The URL to retrieve data from.
  * @param {string} title - The title of the data.
  * @return {Object} - An object containing the retrieved data and metadata.
  */
-const getData = async (ctx, url, title) => {
+const getData = async (url, title) => {
     // Make a GET request to the specified URL
-    const response = await got(url);
-
-    // Extract the response data
-    const data = response.data;
+    const response = await ofetch(url);
 
     // Load the response data into a cheerio object
-    const $ = load(data);
+    const $ = load(response);
     // Get all the list items under the 'ol.dribbbles.group' element
     const list = $('ol.dribbbles.group > li').toArray();
 
     // Process the list items using the ProcessFeed function
-    const result = await ProcessFeed(list, cache);
+    const result = await ProcessFeed(list);
 
     // Return an object containing the retrieved data and metadata
     return {

package/lib/routes/famitsu/category.ts

@@ -106,7 +106,7 @@ async function handler(ctx) {
         },
     });
 
-    const list = (data.pageProps.categoryArticleData as CategoryArticle[])
+    const list = (data.pageProps.categoryArticleDataForPc as CategoryArticle[])
         .filter((item) => !item.advertiserName)
         .map((item) => {
             const publicationDate = item.publishedAt?.slice(0, 7).replace('-', '');

package/lib/routes/fanbox/index.ts

@@ -50,7 +50,7 @@ async function handler(ctx: Context): Promise<Data> {
     }
 
     const postListResponse = (await ofetch(`https://api.fanbox.cc/post.listCreator?creatorId=${creator}&limit=20`, { headers: getHeaders() })) as PostListResponse;
-    const items = await Promise.all(postListResponse.body.items.map((i) => parseItem(i)));
+    const items = await Promise.all(postListResponse.body.map((i) => parseItem(i)));
 
     return {
         title,

package/lib/routes/fanbox/types.ts

@@ -25,10 +25,7 @@ export interface UserInfoResponse {
 }
 
 export interface PostListResponse {
-    body: {
-        items: PostItem[];
-        nextUrl: string | null;
-    };
+    body: PostItem[];
 }
 
 export interface PostDetailResponse {
@@ -88,7 +85,6 @@ interface BasicPost {
                 userId: string;
             };
         }[];
-        nextUrl: string | null;
     };
     coverImageUrl: string | null;
     creatorId: string;

package/lib/routes/fediverse/timeline.ts

@@ -5,6 +5,7 @@ import { parseDate } from '@/utils/parse-date';
 import ofetch from '@/utils/ofetch';
 import { config } from '@/config';
 import ConfigNotFoundError from '@/errors/types/config-not-found';
+import parser from '@/utils/rss-parser';
 
 export const route: Route = {
     path: '/timeline/:account',
@@ -21,7 +22,7 @@ export const route: Route = {
         supportScihub: false,
     },
     name: 'Timeline',
-    maintainers: ['DIYgod'],
+    maintainers: ['DIYgod', 'pseudoyu'],
     handler,
 };
 
@@ -51,12 +52,29 @@ async function handler(ctx) {
             Accept: 'application/jrd+json',
         },
     });
-
     const jsonLink = acc.links.find((link) => link.rel === 'self' && activityPubTypes.has(link.type))?.href;
     const link = acc.links.find((link) => link.rel === 'http://webfinger.net/rel/profile-page')?.href;
+    const officialFeed = await parser.parseURL(`${link}.rss`);
+
+    if (officialFeed) {
+        return {
+            title: `${officialFeed.title} (Fediverse@${account})`,
+            description: officialFeed.description,
+            image: officialFeed.image?.url,
+            link: officialFeed.link,
+            item: officialFeed.items.map((item) => ({
+                title: item.title,
+                description: item.content,
+                link: item.link,
+                pubDate: item.pubDate ? parseDate(item.pubDate) : null,
+                guid: item.guid,
+            })),
+        };
+    }
 
     const self = await ofetch(jsonLink, requestOptions);
 
+    // If RSS feed is not available, fallback to original method
     const outbox = await ofetch(self.outbox, requestOptions);
     const firstOutbox = await ofetch(outbox.first, requestOptions);
 
@@ -88,7 +106,7 @@ async function handler(ctx) {
         image: self.icon?.url || self.image?.url,
         link,
         item: resolvedItems.map((item) => ({
-            title: item.object.content,
+            title: item.object.content.replaceAll(/<[^<]*>/g, ''),
             description: `${item.object.content}\n${item.object.attachment?.map((attachment) => `<img src="${attachment.url}" width="${attachment.width}" height="${attachment.height}" />`).join('\n') || ''}`,
             link: item.object.url,
             pubDate: parseDate(item.published),

package/lib/routes/follow/profile.ts

@@ -1,7 +1,7 @@
 import { ViewType, type Data, type Route } from '@/types';
 import type { Context } from 'hono';
 import ofetch from '@/utils/ofetch';
-import type { FollowResponse, ListSubscription, Profile, Subscription } from './types';
+import type { FollowResponse, InboxSubscription, ListSubscription, Profile, Subscription } from './types';
 import { parse } from 'tldts';
 
 export const route: Route = {
@@ -28,6 +28,8 @@ export const route: Route = {
 
 const isList = (subscription: Subscription): subscription is ListSubscription => 'lists' in subscription;
 
+const isInbox = (subscription: Subscription): subscription is InboxSubscription => 'inboxId' in subscription;
+
 async function handler(ctx: Context): Promise<Data> {
     const uid = ctx.req.param('uid');
     const host = 'https://api.follow.is';
@@ -37,7 +39,7 @@ async function handler(ctx: Context): Promise<Data> {
 
     return {
         title: `${profile.data.name}'s subscriptions`,
-        item: subscriptions.data.map((subscription) => {
+        item: (<Exclude<Subscription, InboxSubscription>[]>subscriptions.data.filter((i) => !isInbox(i))).map((subscription) => {
             if (isList(subscription)) {
                 return {
                     title: subscription.lists.title,

package/lib/routes/follow/types.ts

@@ -3,7 +3,7 @@ export interface FollowResponse<T> {
     data: T;
 }
 
-export type Subscription = FeedSubscription | ListSubscription;
+export type Subscription = FeedSubscription | ListSubscription | InboxSubscription;
 
 export interface Profile {
     id: string;
@@ -66,3 +66,13 @@ export interface ListSubscription extends BaseSubscription {
         view: number;
     };
 }
+
+export interface InboxSubscription extends BaseSubscription {
+    inboxes: {
+        type: 'inbox';
+        id: string;
+        secret: string;
+        title: string;
+    };
+    inboxId: string;
+}

package/lib/routes/github/advisor.ts

@@ -0,0 +1,97 @@
+import { Route } from '@/types';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
+
+export const route: Route = {
+    path: '/advisor/data/:type?/:category?',
+    categories: ['programming'],
+    example: '/github/advisor/data/reviewed/composer',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['github.com/advisories', 'github.com'],
+        },
+    ],
+    name: 'Github Advisory Database RSS',
+    maintainers: ['sd0ric4'],
+    handler,
+    description: `
+| Type | Description | Explanation |
+| --- | --- | --- |
+| reviewed | Reviewed | 已审核 |
+| unreviewed | Unreviewed | 未审核 |
+
+| Category | Description | Explanation |
+| --- | --- | --- |
+| composer | Composer | PHP 依赖管理工具 |
+| go | Go | Go 语言包管理工具 |
+| maven | Maven | Java 项目管理工具 |
+| npm | NPM | Node.js 包管理工具 |
+| nuget | NuGet | .NET 包管理工具 |
+| pip | Pip | Python 包管理工具 |
+| pub | Pub | Dart 包管理工具 |
+| rubygems | RubyGems | Ruby 包管理工具 |
+| rust | Rust | Rust 包管理工具 |
+| erlang | Erlang | Erlang 包管理工具 |
+| actions | Actions | GitHub Actions |
+| swift | Swift | Swift 包管理工具 |`,
+};
+
+async function handler(ctx) {
+    const { category, type } = ctx.req.param();
+
+    const apiRootUrl = 'https://github.com/advisories';
+    const apiUrl = `${apiRootUrl}?query=type%3A${type}+ecosystem%3A${category}`;
+    const currentUrl = `https://github.com/advisories`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+    const $ = load(response.data);
+
+    const list = $('div.Box-row')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('a.Link--primary');
+            const b = item.find('relative-time').attr('datetime');
+            const title = a.text() || 'No title';
+            const link = a.attr('href') || '#';
+            const pubDate = parseDate(b || '');
+
+            return {
+                title,
+                link: `https://github.com${link}`,
+                pubDate,
+                description: '',
+            };
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await ofetch(item.link);
+                const $ = load(response);
+
+                item.description = $('.comment-body').first().html() || '';
+
+                return item;
+            })
+        )
+    );
+    return {
+        title: `GitHub Advisory Database RSS - ${category} - ${type}`,
+        link: currentUrl,
+        item: items,
+    };
+}