rsshub 1.0.0-master.ff3a9ce → 1.0.0-master.ff3ea5a

package/lib/routes/ielts/index.ts

@@ -26,7 +26,7 @@ async function handler() {
     const html = await cache.tryGet(
         targetUrl,
         async () => {
-            const browser = await puppeteer({ stealth: true });
+            const browser = await puppeteer();
             const page = await browser.newPage();
             await page.setRequestInterception(true);
             page.on('request', (request) => {

package/lib/routes/infoq/topic.ts

@@ -1,6 +1,6 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import utils from './utils';
 
 export const route: Route = {
@@ -32,35 +32,38 @@ async function handler(ctx) {
     const infoUrl = 'https://www.infoq.cn/public/v1/topic/getInfo';
     const pageUrl = `https://www.infoq.cn/topic/${paramId}`;
 
-    const infoBody = Number.isNaN(paramId) ? { alias: paramId } : { id: Number.parseInt(paramId) };
+    const infoBody = Number.isNaN(Number(paramId)) ? { alias: paramId } : { id: Number.parseInt(paramId) };
 
-    const info = await got.post(infoUrl, {
+    const info = await ofetch(infoUrl, {
+        method: 'POST',
         headers: {
             Referer: pageUrl,
         },
-        json: infoBody,
+        body: infoBody,
     });
-    const topicName = info.data.data.name;
-    const type = info.data.data.type;
+    const infoData = info.data;
+    const topicName = infoData.name;
 
-    const resp = await got.post(apiUrl, {
+    const resp = await ofetch(apiUrl, {
+        method: 'POST',
         headers: {
             Referer: pageUrl,
         },
-        json: {
+        body: {
+            id: infoData.id,
+            ptype: 0,
             size: ctx.req.query('limit') ? Number(ctx.req.query('limit')) : 30,
-            type,
-            id: info.data.data.id,
+            type: 0,
         },
     });
 
-    const data = resp.data.data;
+    const data = resp.data;
     const items = await utils.ProcessFeed(data, cache);
 
     return {
         title: `InfoQ 话题 - ${topicName}`,
-        description: info.data.data.desc,
-        image: info.data.data.cover,
+        description: infoData.desc,
+        image: infoData.cover,
         link: pageUrl,
         item: items,
     };

package/lib/routes/instagram/web-api/index.ts

@@ -3,7 +3,7 @@ import cache from '@/utils/cache';
 import { CookieJar } from 'tough-cookie';
 import { config } from '@/config';
 import { renderItems } from '../common-utils';
-import { baseUrl, COOKIE_URL, checkLogin, getUserInfo, getUserFeedItems, getTagsFeedItems, getLoggedOutTagsFeedItems, renderGuestItems } from './utils';
+import { baseUrl, COOKIE_URL, checkLogin, getUserInfo, getUserFeedItems, getTagsFeed, renderGuestItems } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
 import ConfigNotFoundError from '@/errors/types/config-not-found';
 
@@ -35,7 +35,7 @@ You may need to setup cookie for a less restrictive rate limit and private profi
 
 async function handler(ctx) {
     // if (!config.instagram || !config.instagram.cookie) {
-    //     throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/deploy/config#route-specific-configurations">relevant config</a>');
+    //     throw new ConfigNotFoundError('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/deploy/config#route-specific-configurations">relevant config</a>');
     // }
     const availableCategories = ['user', 'tags'];
     const { category, key } = ctx.req.param();
@@ -45,7 +45,7 @@ async function handler(ctx) {
     }
 
     let cookieJar = await cache.get('instagram:cookieJar');
-    const wwwClaimV2 = await cache.get('instagram:wwwClaimV2');
+    // const wwwClaimV2 = await cache.get('instagram:wwwClaimV2');
     const cacheMiss = !cookieJar;
 
     if (cacheMiss) {
@@ -59,7 +59,7 @@ async function handler(ctx) {
         cookieJar = CookieJar.fromJSON(cookieJar);
     }
 
-    if (!wwwClaimV2 && cookie && !(await checkLogin(cookieJar, cache))) {
+    if (/* !wwwClaimV2 &&*/ cookie && !(await checkLogin(cookieJar))) {
         throw new ConfigNotFoundError('Invalid cookie');
     }
 
@@ -67,7 +67,7 @@ async function handler(ctx) {
     let items;
     switch (category) {
         case 'user': {
-            const userInfo = await getUserInfo(key, cookieJar, cache);
+            const userInfo = await getUserInfo(key, cookieJar);
 
             // User feed metadata
             const biography = userInfo.biography;
@@ -80,22 +80,26 @@ async function handler(ctx) {
             feedLogo = userInfo.profile_pic_url_hd ?? userInfo.hd_profile_pic_url_info?.url ?? userInfo.profile_pic_url;
             feedLink = `${baseUrl}/${username}`;
 
-            items = cookie ? await getUserFeedItems(id, username, cookieJar, cache) : [...userInfo.edge_felix_video_timeline.edges, ...userInfo.edge_owner_to_timeline_media.edges];
+            items = cookie ? await getUserFeedItems(id, username, cookieJar) : [...userInfo.edge_felix_video_timeline.edges, ...userInfo.edge_owner_to_timeline_media.edges];
 
             break;
         }
         case 'tags': {
+            if (!config.instagram || !config.instagram.cookie) {
+                throw new ConfigNotFoundError('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/deploy/config#route-specific-configurations">relevant config</a>');
+            }
             const tag = key;
 
             feedTitle = `#${tag} - Instagram`;
-            feedLink = `${baseUrl}/explore/tags/${tag}`;
+            feedLink = `${baseUrl}/explore/search/keyword/?q=%23${tag}`;
 
-            if (cookie) {
-                items = await getTagsFeedItems(tag, 'recent', cookieJar, cache.tryGet);
-            } else {
-                const hashTag = await getLoggedOutTagsFeedItems(tag, cookieJar);
-                items = [...hashTag.edge_hashtag_to_media.edges, ...hashTag.edge_hashtag_to_top_posts.edges];
-            }
+            const feedData = await getTagsFeed(tag, cookieJar);
+
+            feedLogo = feedData.profile_pic_url;
+            items = feedData.top.sections.flatMap((section) =>
+                // either media or clips
+                (section.feed_type === 'media' ? section.layout_content.medias : [...section.layout_content.one_by_two_item.clips.items, ...section.layout_content.fill_items]).map((item) => item.media)
+            );
 
             break;
         }

package/lib/routes/instagram/web-api/utils.ts

@@ -1,4 +1,5 @@
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
 import { parseDate } from '@/utils/parse-date';
 import { config } from '@/config';
 import { art } from '@/utils/render';
@@ -6,7 +7,7 @@ import path from 'node:path';
 import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 const baseUrl = 'https://www.instagram.com';
-const COOKIE_URL = 'https://instagram.com';
+const COOKIE_URL = baseUrl;
 
 const getCSRFTokenFromJar = async (cookieJar) => {
     const cookieString = await cookieJar.getCookieString(COOKIE_URL);
@@ -14,45 +15,52 @@ const getCSRFTokenFromJar = async (cookieJar) => {
 };
 
 const getHeaders = async (cookieJar) => ({
-    'X-ASBD-ID': 198387,
-    'X-CSRFToken': await getCSRFTokenFromJar(cookieJar),
-    'X-IG-App-ID': 936_619_743_392_459,
-    'X-IG-WWW-Claim': undefined,
+    'sec-fetch-dest': 'empty',
+    'sec-fetch-mode': 'cors',
+    'sec-fetch-site': 'same-origin',
+    'x-asbd-id': 359341,
+    'x-csrftoken': await getCSRFTokenFromJar(cookieJar),
+    'x-ig-app-id': 936_619_743_392_459,
+    'x-ig-www-claim': '0',
 });
 
-const checkLogin = async (cookieJar, cache) => {
-    const response = await got.post(`${baseUrl}/api/v1/web/fxcal/ig_sso_users/`, {
+const checkLogin = async (cookieJar) => {
+    const response = await ofetch(`${baseUrl}/api/v1/web/fxcal/ig_sso_users/`, {
         // cookieJar,
         headers: {
-            cookie: await cookieJar.getCookieString(COOKIE_URL),
-            ...(await getHeaders(cookieJar)),
-            'X-IG-WWW-Claim': '0',
+            'content-type': 'application/x-www-form-urlencoded',
+            cookie: (await cookieJar.getCookieString(COOKIE_URL)) as string,
+            ...((await getHeaders(cookieJar)) as unknown as Record<string, string>),
+            // 'X-IG-WWW-Claim': '0',
         },
+        method: 'POST',
     });
 
-    const wwwClaimV2 = response.headers['x-ig-set-www-claim'];
+    // const wwwClaimV2 = response.headers['x-ig-set-www-claim'];
 
-    if (wwwClaimV2) {
-        cache.set('instagram:wwwClaimV2', wwwClaimV2);
-    }
+    // if (wwwClaimV2) {
+    //     cache.set('instagram:wwwClaimV2', wwwClaimV2);
+    // }
 
-    return Boolean(response.data.status === 'ok');
+    return Boolean(response.status === 'ok');
 };
 
-const getUserInfo = async (username, cookieJar, cache) => {
+const getUserInfo = async (username, cookieJar) => {
     let webProfileInfo;
     let id = await cache.get(`instagram:getIdByUsername:${username}`);
     let userInfoCache = await cache.get(`instagram:userInfo:${id}`);
+    userInfoCache = userInfoCache && typeof userInfoCache === 'string' ? JSON.parse(userInfoCache) : userInfoCache;
 
     if (!userInfoCache) {
         try {
-            const response = await got(`${baseUrl}/api/v1/users/web_profile_info/`, {
-                cookieJar,
+            const response = await ofetch.raw(`${baseUrl}/api/v1/users/web_profile_info/`, {
+                // cookieJar,
                 headers: {
-                    ...(await getHeaders(cookieJar)),
-                    'X-IG-WWW-Claim': (await cache.get('instagram:wwwClaimV2')) ?? undefined,
+                    cookie: (await cookieJar.getCookieString(COOKIE_URL)) as string,
+                    ...((await getHeaders(cookieJar)) as unknown as Record<string, string>),
+                    // 'X-IG-WWW-Claim': (await cache.get('instagram:wwwClaimV2')) ?? undefined,
                 },
-                searchParams: {
+                query: {
                     username,
                 },
             });
@@ -60,7 +68,7 @@ const getUserInfo = async (username, cookieJar, cache) => {
                 throw new ConfigNotFoundError('Invalid cookie');
             }
 
-            webProfileInfo = response.data.data.user;
+            webProfileInfo = response._data.data.user;
             id = webProfileInfo.id;
 
             await cache.set(`instagram:getIdByUsername:${username}`, id, 31_536_000); // 1 year since it will never change
@@ -73,24 +81,22 @@ const getUserInfo = async (username, cookieJar, cache) => {
         }
     }
 
-    userInfoCache = typeof userInfoCache === 'string' ? JSON.parse(userInfoCache) : userInfoCache;
-
     return userInfoCache || webProfileInfo;
 };
 
-const getUserFeedItems = (id, username, cookieJar, cache) =>
+const getUserFeedItems = (id, username, cookieJar) =>
     cache.tryGet(
         `instagram:feed:${id}`,
         async () => {
-            const response = await got(`${baseUrl}/api/v1/feed/user/${username}/username/`, {
+            const response = await ofetch.raw(`${baseUrl}/api/v1/feed/user/${username}/username/`, {
                 // cookieJar,
                 headers: {
-                    cookie: await cookieJar.getCookieString(COOKIE_URL),
-                    ...(await getHeaders(cookieJar)),
+                    cookie: (await cookieJar.getCookieString(COOKIE_URL)) as string,
+                    ...((await getHeaders(cookieJar)) as unknown as Record<string, string>),
                     // 401 Unauthorized if cookie does not match with IP
-                    'X-IG-WWW-Claim': await cache.get('instagram:wwwClaimV2'),
+                    // 'X-IG-WWW-Claim': await cache.get('instagram:wwwClaimV2'),
                 },
-                searchParams: {
+                query: {
                     count: 30,
                 },
             });
@@ -99,56 +105,33 @@ const getUserFeedItems = (id, username, cookieJar, cache) =>
                 Please also check if your account is being blocked by Instagram.`);
             }
 
-            return response.data.items;
+            return response._data.items;
         },
         config.cache.routeExpire,
         false
     );
 
-const getTagsFeedItems = (tag, tab, cookieJar, tryGet) =>
-    tryGet(
+const getTagsFeed = (tag, cookieJar) =>
+    cache.tryGet(
         `instagram:tags:${tag}`,
         async () => {
-            const response = await got(`${baseUrl}/api/v1/tags/web_info/`, {
+            const response = await ofetch(`${baseUrl}/api/v1/tags/web_info/`, {
                 // cookieJar, cookieJar is behaving weirdly here, so we use cookie header instead
                 headers: {
-                    cookie: await cookieJar.getCookieString(COOKIE_URL),
-                    ...(await getHeaders(cookieJar)),
+                    cookie: (await cookieJar.getCookieString(COOKIE_URL)) as string,
+                    ...((await getHeaders(cookieJar)) as unknown as Record<string, string>),
                 },
-                searchParams: {
+                query: {
                     tag_name: tag,
                 },
             });
 
-            return response.data.data[tab].sections.flatMap((section) => section.layout_content.medias.map((media) => media.media));
+            return response.data;
         },
         config.cache.routeExpire,
         false
     );
 
-const getLoggedOutTagsFeedItems = async (tag, cookieJar) => {
-    const response = await got(`${baseUrl}/api/v1/tags/logged_out_web_info/`, {
-        // cookieJar, cookieJar is behaving weirdly here, so we use cookie header instead
-        headers: {
-            cookie: await cookieJar.getCookieString(COOKIE_URL),
-            ...(await getHeaders(cookieJar)),
-            'X-IG-WWW-Claim': '0',
-        },
-        searchParams: {
-            tag_name: tag,
-        },
-    });
-
-    const cookies = response.headers['set-cookie'];
-    if (cookies) {
-        for await (const cookie of cookies) {
-            await cookieJar.setCookie(cookie, COOKIE_URL);
-        }
-    }
-
-    return response.data.data.hashtag;
-};
-
 const renderGuestItems = (items) => {
     const renderVideo = (node, summary) =>
         art(path.join(__dirname, '../templates/video.art'), {
@@ -212,4 +195,4 @@ const renderGuestItems = (items) => {
     });
 };
 
-export { baseUrl, COOKIE_URL, checkLogin, getUserInfo, getUserFeedItems, getTagsFeedItems, getLoggedOutTagsFeedItems, renderGuestItems };
+export { baseUrl, COOKIE_URL, checkLogin, getUserInfo, getUserFeedItems, getTagsFeed, renderGuestItems };

package/lib/routes/jamesclear/book-summaries.ts

@@ -0,0 +1,40 @@
+import { Data, Route, ViewType } from '@/types';
+import { rootUrl, fetchContent, processItem } from './utils';
+
+export const route: Route = {
+    path: '/book-summaries',
+    view: ViewType.Articles,
+    categories: ['blog'],
+    example: '/jamesclear/book-summaries',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['jamesclear.com/book-summaries'],
+            target: '/book-summaries',
+        },
+    ],
+    name: 'Book Summaries',
+    maintainers: ['Rjnishant530'],
+    handler,
+};
+
+async function handler(): Promise<Data> {
+    const summaries = await fetchContent('book-summaries');
+    const items = summaries.map((item) => processItem(item));
+
+    return {
+        title: 'James Clear - Book Summaries',
+        description: 'Book summaries by James Clear',
+        link: `${rootUrl}/book-summaries`,
+        item: items,
+        language: 'en',
+        icon: `${rootUrl}/favicon.ico`,
+    };
+}

package/lib/routes/jamesclear/great-speeches.ts

@@ -0,0 +1,40 @@
+import { Data, Route, ViewType } from '@/types';
+import { rootUrl, fetchContent, processItem } from './utils';
+
+export const route: Route = {
+    path: '/great-speeches',
+    view: ViewType.Articles,
+    categories: ['blog'],
+    example: '/jamesclear/great-speeches',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['jamesclear.com/great-speeches'],
+            target: '/great-speeches',
+        },
+    ],
+    name: 'Great Speeches',
+    maintainers: ['Rjnishant530'],
+    handler,
+};
+
+async function handler(): Promise<Data> {
+    const speeches = await fetchContent('great-speeches');
+    const items = speeches.map((item) => processItem(item));
+
+    return {
+        title: 'James Clear - Great Speeches',
+        description: 'Collection of great speeches curated by James Clear',
+        link: `${rootUrl}/great-speeches`,
+        item: items,
+        language: 'en',
+        icon: `${rootUrl}/favicon.ico`,
+    };
+}

package/lib/routes/jamesclear/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'James Clear',
+    url: 'jamesclear.com',
+    categories: ['blog'],
+    lang: 'en',
+};

package/lib/routes/jamesclear/quotes.ts

@@ -0,0 +1,40 @@
+import { Data, Route, ViewType } from '@/types';
+import { rootUrl, fetchContent, processItem } from './utils';
+
+export const route: Route = {
+    path: '/quotes',
+    view: ViewType.Articles,
+    categories: ['blog'],
+    example: '/jamesclear/quotes',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['jamesclear.com/quotes'],
+            target: '/quotes',
+        },
+    ],
+    name: 'Quotes',
+    maintainers: ['Rjnishant530'],
+    handler,
+};
+
+async function handler(): Promise<Data> {
+    const quotes = await fetchContent('quotes');
+    const items = quotes.map((item) => processItem(item));
+
+    return {
+        title: 'James Clear - Quotes',
+        description: 'Quotes from James Clear',
+        link: `${rootUrl}/quotes`,
+        item: items,
+        language: 'en',
+        icon: `${rootUrl}/favicon.ico`,
+    };
+}

package/lib/routes/jamesclear/three-two-one.ts

@@ -0,0 +1,41 @@
+import { Data, Route, ViewType } from '@/types';
+import { rootUrl, fetchContent, processItem } from './utils';
+
+export const route: Route = {
+    path: '/3-2-1',
+    view: ViewType.Articles,
+    categories: ['blog'],
+    example: '/jamesclear/3-2-1',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['jamesclear.com/3-2-1'],
+            target: '/3-2-1',
+        },
+    ],
+    name: '3-2-1 Newsletter',
+    maintainers: ['Rjnishant530'],
+    handler,
+};
+
+async function handler(): Promise<Data> {
+    const newsletters = await fetchContent('3-2-1');
+    const items = newsletters.map((item) => processItem(item));
+
+    return {
+        title: 'James Clear - 3-2-1 Newsletter',
+        description: '3 ideas, 2 quotes, and 1 question to consider each week',
+        link: `${rootUrl}/3-2-1`,
+        item: items,
+        language: 'en',
+        image: `${rootUrl}/wp-content/uploads/2021/04/3-2-1-Featured-Image.png`,
+        icon: `${rootUrl}/favicon.ico`,
+    };
+}

package/lib/routes/jamesclear/utils.ts

@@ -0,0 +1,22 @@
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { DataItem } from '@/types';
+
+export const rootUrl = 'https://jamesclear.com';
+export const apiUrl = `${rootUrl}/wp-json/wp/v2`;
+
+export async function fetchContent(endpoint: string) {
+    const response = await ofetch(`${apiUrl}/${endpoint}`);
+    return response;
+}
+
+export function processItem(item: any): DataItem {
+    return {
+        title: item.title.rendered,
+        link: item.link,
+        description: item.content.rendered,
+        pubDate: parseDate(item.date_gmt),
+        author: 'James Clear',
+        guid: item.guid.rendered,
+    };
+}

package/lib/routes/japanpost/utils.ts

@@ -57,10 +57,10 @@ const utils = {
 
         switch (l) {
             case 'ja':
-                customFormat = dayjs(t, formatJaDate, true).isValid() ? formatJaDate : (dayjs(t, formatJaDateTime, true).isValid() ? formatJaDateTime : undefined);
+                customFormat = dayjs(t, formatJaDate, true).isValid() ? formatJaDate : dayjs(t, formatJaDateTime, true).isValid() ? formatJaDateTime : undefined;
                 break;
             case 'en':
-                customFormat = dayjs(t, formatEnDate, true).isValid() ? formatEnDate : (dayjs(t, formatEnDateTime, true).isValid() ? formatEnDateTime : undefined);
+                customFormat = dayjs(t, formatEnDate, true).isValid() ? formatEnDate : dayjs(t, formatEnDateTime, true).isValid() ? formatEnDateTime : undefined;
                 break;
         }
 

package/lib/routes/javbus/index.ts

@@ -37,6 +37,9 @@ export const route: Route = {
             description: 'Any path of list page on javbus',
         },
     },
+    features: {
+        nsfw: true,
+    },
 };
 
 async function handler(ctx) {

package/lib/routes/javdb/tags.ts

@@ -19,6 +19,7 @@ export const route: Route = {
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
+        nsfw: true,
     },
     radar: [
         {

package/lib/routes/javlibrary/star.ts

@@ -14,6 +14,7 @@ export const route: Route = {
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
+        nsfw: true,
     },
     name: 'Videos by star',
     maintainers: ['nczitzk'],

package/lib/routes/javtiful/actress.ts

@@ -17,6 +17,9 @@ export const route: Route = {
             target: '/actress/:id',
         },
     ],
+    features: {
+        nsfw: true,
+    },
 };
 
 async function handler(ctx): Promise<Data> {

package/lib/routes/javtiful/channel.ts

@@ -17,6 +17,9 @@ export const route: Route = {
             target: '/channel/:id',
         },
     ],
+    features: {
+        nsfw: true,
+    },
 };
 
 async function handler(ctx): Promise<Data> {

package/lib/routes/javtrailers/casts.ts

@@ -18,6 +18,9 @@ export const route: Route = {
     maintainers: ['TonyRL'],
     url: 'javtrailers.com/casts',
     handler,
+    features: {
+        nsfw: true,
+    },
 };
 
 async function handler(ctx) {