rsshub 1.0.0-master.fd5a888 → 1.0.0-master.fd5cc68

package/lib/routes/threads/index.ts

@@ -1,10 +1,9 @@
 import { Route, ViewType } from '@/types';
-import ofetch from '@/utils/ofetch';
 import { parseDate } from '@/utils/parse-date';
-import { REPLIES_QUERY, THREADS_QUERY, apiUrl, threadUrl, profileUrl, extractTokens, makeHeader, getUserId, buildContent } from './utils';
-import { destr } from 'destr';
-import cache from '@/utils/cache';
-import { config } from '@/config';
+import { threadUrl, profileUrl, extractTokens, getUserId, buildContent } from './utils';
+import { JSDOM } from 'jsdom';
+import { JSONPath } from 'jsonpath-plus';
+import ofetch from '@/utils/ofetch';
 
 export const route: Route = {
     path: '/:user/:routeParams?',
@@ -25,13 +24,7 @@ Specify options (in the format of query string) in parameter \`routeParams\` to
 | \`showAuthorAvatarInDesc\`       | Show avatar of author in description (RSS body) (Not recommended if your RSS reader extracts images from description)        | \`0\`/\`1\`/\`true\`/\`false\` | \`falseP\`    |
 | \`showEmojiForQuotesAndReply\`   | Use "🔁" instead of "QT", "↩️" instead of "Re"                                                                               | \`0\`/\`1\`/\`true\`/\`false\` | \`true\`      |
 | \`showQuotedInTitle\`            | Show quoted tweet in title                                                                                                   | \`0\`/\`1\`/\`true\`/\`false\` | \`true\`      |
-| \`replies\`                      | Show replies                                                                                                                 | \`0\`/\`1\`/\`true\`/\`false\` | \`true\`      |
-
-Specify different option values than default values to improve readability. The URL
-
-\`\`\`
-https://rsshub.app/threads/zuck/showAuthorInTitle=1&showAuthorInDesc=1&showQuotedAuthorAvatarInDesc=1&showAuthorAvatarInDesc=1&showEmojiForQuotesAndReply=1&showQuotedInTitle=1
-\`\`\``,
+| \`replies\`                      | Show replies                                                                                                                 | \`0\`/\`1\`/\`true\`/\`false\` | \`true\`      |`,
         },
     },
     name: 'User timeline',
@@ -42,10 +35,10 @@ https://rsshub.app/threads/zuck/showAuthorInTitle=1&showAuthorInDesc=1&showQuote
 async function handler(ctx) {
     const { user, routeParams } = ctx.req.param();
     const { lsd } = await extractTokens(user);
-    const userId = await getUserId(user, lsd);
+    const userId = await getUserId(user);
 
     const params = new URLSearchParams(routeParams);
-    const debugJson = {
+    const debugJson: any = {
         params: routeParams,
         lsd,
     };
@@ -60,48 +53,59 @@ async function handler(ctx) {
         replies: params.get('replies') ?? false,
     };
 
-    const threadsResponse = await cache.tryGet(
-        `threads:${userId}:${options.replies}`,
-        () =>
-            ofetch(apiUrl, {
-                method: 'POST',
-                headers: {
-                    ...makeHeader(user, lsd),
-                    'content-type': 'application/x-www-form-urlencoded',
-                },
-                body: new URLSearchParams({
-                    lsd,
-                    variables: JSON.stringify({ userID: userId }),
-                    doc_id: String(options.replies ? REPLIES_QUERY : THREADS_QUERY),
-                }).toString(),
-                parseResponse: (txt) => destr(txt),
-            }),
-        config.cache.routeExpire,
-        false
-    );
+    const response = await ofetch(profileUrl(user), {
+        headers: {
+            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1',
+            Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'Accept-Encoding': 'gzip, br',
+            'Accept-Language': 'zh-CN,zh;q=0.9',
+            'Cache-Control': 'no-cache',
+            Pragma: 'no-cache',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-User': '?1',
+            'Upgrade-Insecure-Requests': '1',
+        },
+    });
+
+    const dom = new JSDOM(response);
+
+    let threadsData: ThreadItem[] | null = null;
+    for (const el of dom.window.document.querySelectorAll('script[data-sjs]')) {
+        try {
+            const data = JSONPath({
+                path: '$..thread_items[0]',
+                json: JSON.parse(el.textContent || ''),
+            });
+
+            if (data?.length > 0) {
+                threadsData = data as ThreadItem[];
+                break;
+            }
+        } catch {
+            // Skip invalid JSON
+        }
+    }
+
+    if (!threadsData) {
+        throw new Error('Failed to fetch thread data');
+    }
 
     debugJson.profileId = userId;
-    debugJson.response = {
-        response: threadsResponse,
-    };
+    debugJson.response = { response: threadsData };
 
-    const threads = threadsResponse?.data?.mediaData?.threads || [];
-    const userData = threadsResponse?.data?.mediaData?.threads?.[0]?.thread_items?.[0]?.post?.user || {};
-
-    const items = threads.flatMap((thread) =>
-        thread.thread_items
-            .filter((item) => user === item.post.user?.username)
-            .map((item) => {
-                const { title, description } = buildContent(item, options);
-                return {
-                    author: user,
-                    title,
-                    description,
-                    pubDate: parseDate(item.post.taken_at, 'X'),
-                    link: threadUrl(item.post.code),
-                };
-            })
-    );
+    const userData: ThreadUser = threadsData[0]?.post?.user || { username: user, profile_pic_url: '' };
+
+    const items = threadsData
+        .filter((item) => user === item.post.user?.username)
+        .map((item) => ({
+            author: user,
+            title: buildContent(item, options).title,
+            description: buildContent(item, options).description,
+            pubDate: parseDate(item.post.taken_at, 'X'),
+            link: threadUrl(item.post.code),
+        }));
 
     debugJson.items = items;
     ctx.set('json', debugJson);
@@ -110,7 +114,22 @@ async function handler(ctx) {
         title: `${user} (@${user}) on Threads`,
         link: profileUrl(user),
         image: userData?.profile_pic_url,
-        // description: userData.biography,
         item: items,
     };
 }
+
+interface ThreadUser {
+    username: string;
+    profile_pic_url: string;
+}
+
+interface ThreadItem {
+    post: {
+        user?: ThreadUser;
+        taken_at: number;
+        code: string;
+        caption?: {
+            text: string;
+        };
+    };
+}

package/lib/routes/threads/utils.ts

@@ -1,97 +1,91 @@
-import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import dayjs from 'dayjs';
 import cache from '@/utils/cache';
-import { destr } from 'destr';
 import NotFoundError from '@/errors/types/not-found';
+import ofetch from '@/utils/ofetch';
+import { JSDOM } from 'jsdom';
+import { JSONPath } from 'jsonpath-plus';
 
 const profileUrl = (user: string) => `https://www.threads.net/@${user}`;
 const threadUrl = (code: string) => `https://www.threads.net/t/${code}`;
-const instagramUrl = (user: string) => `https://i.instagram.com/api/v1/users/web_profile_info/?username=${user}`;
 
-const apiUrl = 'https://www.threads.net/api/graphql';
-// const PROFILE_QUERY = 23_996_318_473_300_828; // no longer works
-const THREADS_QUERY = 6_232_751_443_445_612;
-const REPLIES_QUERY = 6_307_072_669_391_286;
-const USER_AGENT = 'Barcelona 289.0.0.77.109 Android';
-const appId = '238260118697367';
-const asbdId = '129477';
+const USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1';
 
 const extractTokens = async (user): Promise<{ lsd: string }> => {
     const response = await ofetch(profileUrl(user), {
         headers: {
             'User-Agent': USER_AGENT,
-            'X-IG-App-ID': appId,
+            Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'Accept-Encoding': 'gzip, br',
+            'Accept-Language': 'zh-CN,zh;q=0.9',
+            'Cache-Control': 'no-cache',
+            Pragma: 'no-cache',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-User': '?1',
+            'Upgrade-Insecure-Requests': '1',
         },
     });
-    const $ = load(response);
 
+    const $ = load(response);
     const data = $('script:contains("LSD"):first').text();
-
     const lsd = data.match(/"LSD",\[],{"token":"([\w@-]+)"},/)?.[1];
+
     if (!lsd) {
         throw new NotFoundError('LSD token not found');
     }
 
-    // const userId = data.match(/{"user_id":"(\d+)"},/)?.[1];
-
-    const ret = { lsd };
-    return ret;
+    return { lsd };
 };
 
-const makeHeader = (user: string, lsd: string) => ({
-    Accept: '*/*',
-    Host: 'www.threads.net',
-    Origin: 'https://www.threads.net',
-    Referer: profileUrl(user),
-    'User-Agent': USER_AGENT,
-    'X-FB-LSD': lsd,
-    'X-IG-App-ID': appId,
-    'Sec-Fetch-Site': 'same-origin',
-});
-
-// the formal way always reachs the rate limit, so use instagram api to get user id instead
-const getUserId = (user: string, lsd: string): Promise<string> =>
-    cache.tryGet(`threads:userId:${user}`, async () => {
-        const pathName = `/@${user}`;
-        const payload: any = {
-            'route_urls[0]': pathName,
-            __a: '1',
-            __comet_req: '29',
-            lsd,
-        };
-        const response = await ofetch('https://www.threads.net/ajax/bulk-route-definitions/', {
-            method: 'POST',
-            headers: {
-                ...makeHeader(user, lsd),
-                'content-type': 'application/x-www-form-urlencoded',
-                'X-ASBD-ID': asbdId,
-            },
-            body: new URLSearchParams(payload).toString(),
-            parseResponse: (txt) => destr(txt.slice(9)), // remove "for (;;);"
-        });
-
-        let userId = response.payload.payloads[pathName].result.exports.rootView.props.user_id;
-
-        if (!userId) {
-            const fallbackResponse = await ofetch(instagramUrl(user), {
-                headers: makeHeader(user, lsd),
+const getUserId = (user: string): Promise<string> =>
+    cache
+        .tryGet(`threads:userId:${user}`, async () => {
+            const response = await ofetch(profileUrl(user), {
+                headers: {
+                    'User-Agent': USER_AGENT,
+                    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+                    'Accept-Encoding': 'gzip, br',
+                    'Accept-Language': 'zh-CN,zh;q=0.9',
+                    'Cache-Control': 'no-cache',
+                    Pragma: 'no-cache',
+                    'Sec-Fetch-Dest': 'document',
+                    'Sec-Fetch-Mode': 'navigate',
+                    'Sec-Fetch-Site': 'none',
+                    'Sec-Fetch-User': '?1',
+                    'Upgrade-Insecure-Requests': '1',
+                },
             });
 
-            if (!fallbackResponse?.data?.user) {
-                throw new NotFoundError('Instagram getUser API response is invalid');
+            const dom = new JSDOM(response);
+
+            for (const el of dom.window.document.querySelectorAll('script[data-sjs]')) {
+                try {
+                    const data = JSONPath({
+                        path: '$..user_id',
+                        json: JSON.parse(el.textContent || ''),
+                    });
+
+                    if (data?.[0]) {
+                        return data[0];
+                    }
+                } catch {
+                    // Skip invalid JSON
+                }
             }
 
-            userId = fallbackResponse.data.user.id;
-            if (!userId) {
-                throw new NotFoundError('User ID not found in Instagram getUser API response');
+            throw new NotFoundError('User ID not found');
+        })
+        .then((result): string => {
+            if (!result || typeof result !== 'string') {
+                throw new TypeError('Invalid user ID type');
             }
-        }
-
-        return userId;
-    });
+            return result;
+        });
 
 const hasMedia = (post) => post.image_versions2 || post.carousel_media || post.video_versions;
+
 const buildMedia = (post) => {
     let html = '';
 
@@ -99,25 +93,13 @@ const buildMedia = (post) => {
         for (const media of post.carousel_media) {
             const firstImage = media.image_versions2?.candidates[0];
             const firstVideo = media.video_versions?.[0];
-            if (firstVideo) {
-                html += `<video controls autoplay loop poster="${firstImage.url}">`;
-                html += `<source src="${firstVideo.url}"/>`;
-                html += '</video>';
-            } else {
-                html += `<img src="${firstImage.url}"/>`;
-            }
+            html += firstVideo ? `<video controls autoplay loop poster="${firstImage.url}"><source src="${firstVideo.url}"/></video>` : `<img src="${firstImage.url}"/>`;
         }
     } else {
         const mainImage = post.image_versions2?.candidates?.[0];
         const mainVideo = post.video_versions?.[0];
         if (mainImage) {
-            if (mainVideo) {
-                html += `<video controls autoplay loop poster="${mainImage.url}">`;
-                html += `<source src="${mainVideo.url}"/>`;
-                html += '</video>';
-            } else {
-                html += `<img src="${mainImage.url}"/>`;
-            }
+            html += mainVideo ? `<video controls autoplay loop poster="${mainImage.url}"><source src="${mainVideo.url}"/></video>` : `<img src="${mainImage.url}"/>`;
         }
     }
 
@@ -181,4 +163,4 @@ const buildContent = (item, options) => {
     return { title, description };
 };
 
-export { apiUrl, profileUrl, threadUrl, THREADS_QUERY, REPLIES_QUERY, USER_AGENT, extractTokens, getUserId, makeHeader, hasMedia, buildMedia, buildContent };
+export { profileUrl, threadUrl, extractTokens, getUserId, buildContent };

package/lib/routes/tiddlywiki/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'TiddlyWiki',
+    url: 'tiddlywiki.com',
+    description: '',
+    lang: 'en',
+};

package/lib/routes/tiddlywiki/releases.ts

@@ -0,0 +1,84 @@
+import { Route } from '@/types';
+
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/releases',
+    categories: ['program-update'],
+    example: '/tiddlywiki/releases',
+    url: 'tiddlywiki.com',
+    name: 'Releases',
+    maintainers: ['p3psi-boo'],
+    radar: [
+        {
+            source: ['github.com/TiddlyWiki/TiddlyWiki5'],
+            target: '/releases',
+        },
+        {
+            source: ['tiddlywiki.com'],
+            target: '/releases',
+        },
+    ],
+    handler,
+};
+
+async function handler() {
+    const tagListUrl = 'https://github.com/TiddlyWiki/TiddlyWiki5/releases.atom';
+
+    const response = await got({
+        method: 'get',
+        url: tagListUrl,
+    });
+
+    const $ = load(response.data);
+
+    const alist = $('entry');
+
+    const versionList = alist
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const text = item.find('title').text();
+            const date = item.find('updated').text();
+            // 使用正则提取 v5.3.6 格式
+            const version = text.match(/v\d+\.\d+\.\d+/)?.[0];
+            return {
+                version,
+                pubDate: parseDate(date),
+            };
+        })
+        .filter((item) => item.version);
+
+    const items = await Promise.all(
+        versionList.map((item) => {
+            const _version = item.version.slice(1);
+            const url = `https://tiddlywiki.com/static/Release%2520${_version}.html`;
+            return cache.tryGet(url, async () => {
+                const response = await got({
+                    method: 'get',
+                    url,
+                });
+
+                const $ = load(response.data);
+
+                const description = $('.tc-tiddler-body').html();
+
+                return {
+                    title: item.version,
+                    link: url,
+                    pubDate: item.pubDate,
+                    description,
+                };
+            });
+        })
+    );
+
+    return {
+        title: 'TiddlyWiki Releases',
+        link: 'https://tiddlywiki.com/static/Releases.html',
+        item: items,
+    };
+}

package/lib/routes/tiktok/user.ts

@@ -53,7 +53,7 @@ async function handler(ctx) {
                 waitUntil: 'networkidle0',
             });
             const SIGI_STATE = await page.evaluate(() => window.SIGI_STATE);
-            browser.close();
+            await browser.close();
 
             const lang = SIGI_STATE.AppContext.lang;
             const SharingMetaState = SIGI_STATE.SharingMetaState;

package/lib/routes/timednews/news.ts

@@ -55,7 +55,7 @@ const PATH_LIST = {
 
 export const route: Route = {
     path: '/news/:type?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/timednews/news',
     parameters: { type: '子分类，见下表，默认为全部' },
     features: {
@@ -71,9 +71,9 @@ export const route: Route = {
     handler,
     description: `子分类
 
-  | 全部 | 时政           | 财经    | 科技       | 社会   | 体娱   | 国际          | 美国 | 中国 | 欧洲   | 评论     |
-  | ---- | -------------- | ------- | ---------- | ------ | ------ | ------------- | ---- | ---- | ------ | -------- |
-  | all  | currentAffairs | finance | technology | social | sports | international | usa  | cn   | europe | comments |`,
+| 全部 | 时政           | 财经    | 科技       | 社会   | 体娱   | 国际          | 美国 | 中国 | 欧洲   | 评论     |
+| ---- | -------------- | ------- | ---------- | ------ | ------ | ------------- | ---- | ---- | ------ | -------- |
+| all  | currentAffairs | finance | technology | social | sports | international | usa  | cn   | europe | comments |`,
 };
 
 async function handler(ctx) {

package/lib/routes/tingshuitz/changsha.ts

@@ -22,10 +22,10 @@ export const route: Route = {
     handler,
     description: `可能仅限于中国大陆服务器访问，以实际情况为准。
 
-  | channelId | 分类     |
-  | --------- | -------- |
-  | 78        | 计划停水 |
-  | 157       | 抢修停水 |`,
+| channelId | 分类     |
+| --------- | -------- |
+| 78        | 计划停水 |
+| 157       | 抢修停水 |`,
 };
 
 async function handler(ctx) {

package/lib/routes/tju/cic/index.ts

@@ -33,8 +33,8 @@ export const route: Route = {
     maintainers: ['AlanZeng423', 'SuperPung'],
     handler,
     description: `| College News | Notification | TJU Forum for CIC |
-  | :----------: | :----------: | :---------------: |
-  |     news     | notification |       forum       |`,
+| :----------: | :----------: | :---------------: |
+|     news     | notification |       forum       |`,
 };
 
 async function handler(ctx) {

package/lib/routes/tju/news/index.ts

@@ -35,8 +35,8 @@ export const route: Route = {
     maintainers: ['AlanZeng423', 'SuperPung'],
     handler,
     description: `| Focus on TJU | General News | Internal News | Media Report | Pictures of TJU |
-  | :----------: | :----------: | :-----------: | :----------: | :-------------: |
-  |     focus    |    general   |    internal   |     media    |     picture     |`,
+| :----------: | :----------: | :-----------: | :----------: | :-------------: |
+|     focus    |    general   |    internal   |     media    |     picture     |`,
 };
 
 async function handler(ctx) {

package/lib/routes/tju/oaa/index.ts

@@ -40,8 +40,8 @@ export const route: Route = {
     maintainers: ['AlanZeng423', 'AmosChenYQ', 'SuperPung'],
     handler,
     description: `| News | Notification |
-  | :--: | :----------: |
-  | news | notification |`,
+| :--: | :----------: |
+| news | notification |`,
 };
 
 async function handler(ctx) {

package/lib/routes/tju/yzb/index.ts

@@ -36,8 +36,8 @@ export const route: Route = {
     maintainers: ['SuperPung'],
     handler,
     description: `| School-level Notice | Master | Doctor | On-the-job Degree |
-  | :-----------------: | :----: | :----: | :---------------: |
-  |        notice       | master | doctor |        job        |`,
+| :-----------------: | :----: | :----: | :---------------: |
+|        notice       | master | doctor |        job        |`,
 };
 
 async function handler(ctx) {

package/lib/routes/tkww/index.ts

@@ -28,7 +28,7 @@ export const route: Route = {
     ],
     handler,
     description: `
-:::tip
+::: tip
 欄目可用\`名稱\`或對應網頁的\`path\`，
 如 \`https://www.tkww.hk/hong_kong\` 的欄目可以填\`香港\`或是\`hong_kong\`
 而 \`https://www.tkww.hk/china/shanghai\` 的欄目則需填\`china/shanghai\`