rsshub 1.0.0-master.f6cb490 → 1.0.0-master.f6f0273

package/lib/routes/threads/utils.ts

@@ -1,97 +1,91 @@
-import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import dayjs from 'dayjs';
 import cache from '@/utils/cache';
-import { destr } from 'destr';
 import NotFoundError from '@/errors/types/not-found';
+import ofetch from '@/utils/ofetch';
+import { JSDOM } from 'jsdom';
+import { JSONPath } from 'jsonpath-plus';
 
 const profileUrl = (user: string) => `https://www.threads.net/@${user}`;
 const threadUrl = (code: string) => `https://www.threads.net/t/${code}`;
-const instagramUrl = (user: string) => `https://i.instagram.com/api/v1/users/web_profile_info/?username=${user}`;
 
-const apiUrl = 'https://www.threads.net/api/graphql';
-// const PROFILE_QUERY = 23_996_318_473_300_828; // no longer works
-const THREADS_QUERY = 6_232_751_443_445_612;
-const REPLIES_QUERY = 6_307_072_669_391_286;
-const USER_AGENT = 'Barcelona 289.0.0.77.109 Android';
-const appId = '238260118697367';
-const asbdId = '129477';
+const USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1';
 
 const extractTokens = async (user): Promise<{ lsd: string }> => {
     const response = await ofetch(profileUrl(user), {
         headers: {
             'User-Agent': USER_AGENT,
-            'X-IG-App-ID': appId,
+            Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'Accept-Encoding': 'gzip, br',
+            'Accept-Language': 'zh-CN,zh;q=0.9',
+            'Cache-Control': 'no-cache',
+            Pragma: 'no-cache',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-User': '?1',
+            'Upgrade-Insecure-Requests': '1',
         },
     });
-    const $ = load(response);
 
+    const $ = load(response);
     const data = $('script:contains("LSD"):first').text();
-
     const lsd = data.match(/"LSD",\[],{"token":"([\w@-]+)"},/)?.[1];
+
     if (!lsd) {
         throw new NotFoundError('LSD token not found');
     }
 
-    // const userId = data.match(/{"user_id":"(\d+)"},/)?.[1];
-
-    const ret = { lsd };
-    return ret;
+    return { lsd };
 };
 
-const makeHeader = (user: string, lsd: string) => ({
-    Accept: '*/*',
-    Host: 'www.threads.net',
-    Origin: 'https://www.threads.net',
-    Referer: profileUrl(user),
-    'User-Agent': USER_AGENT,
-    'X-FB-LSD': lsd,
-    'X-IG-App-ID': appId,
-    'Sec-Fetch-Site': 'same-origin',
-});
-
-// the formal way always reachs the rate limit, so use instagram api to get user id instead
-const getUserId = (user: string, lsd: string): Promise<string> =>
-    cache.tryGet(`threads:userId:${user}`, async () => {
-        const pathName = `/@${user}`;
-        const payload: any = {
-            'route_urls[0]': pathName,
-            __a: '1',
-            __comet_req: '29',
-            lsd,
-        };
-        const response = await ofetch('https://www.threads.net/ajax/bulk-route-definitions/', {
-            method: 'POST',
-            headers: {
-                ...makeHeader(user, lsd),
-                'content-type': 'application/x-www-form-urlencoded',
-                'X-ASBD-ID': asbdId,
-            },
-            body: new URLSearchParams(payload).toString(),
-            parseResponse: (txt) => destr(txt.slice(9)), // remove "for (;;);"
-        });
-
-        let userId = response.payload.payloads[pathName].result.exports.rootView.props.user_id;
-
-        if (!userId) {
-            const fallbackResponse = await ofetch(instagramUrl(user), {
-                headers: makeHeader(user, lsd),
+const getUserId = (user: string): Promise<string> =>
+    cache
+        .tryGet(`threads:userId:${user}`, async () => {
+            const response = await ofetch(profileUrl(user), {
+                headers: {
+                    'User-Agent': USER_AGENT,
+                    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+                    'Accept-Encoding': 'gzip, br',
+                    'Accept-Language': 'zh-CN,zh;q=0.9',
+                    'Cache-Control': 'no-cache',
+                    Pragma: 'no-cache',
+                    'Sec-Fetch-Dest': 'document',
+                    'Sec-Fetch-Mode': 'navigate',
+                    'Sec-Fetch-Site': 'none',
+                    'Sec-Fetch-User': '?1',
+                    'Upgrade-Insecure-Requests': '1',
+                },
             });
 
-            if (!fallbackResponse?.data?.user) {
-                throw new NotFoundError('Instagram getUser API response is invalid');
+            const dom = new JSDOM(response);
+
+            for (const el of dom.window.document.querySelectorAll('script[data-sjs]')) {
+                try {
+                    const data = JSONPath({
+                        path: '$..user_id',
+                        json: JSON.parse(el.textContent || ''),
+                    });
+
+                    if (data?.[0]) {
+                        return data[0];
+                    }
+                } catch {
+                    // Skip invalid JSON
+                }
             }
 
-            userId = fallbackResponse.data.user.id;
-            if (!userId) {
-                throw new NotFoundError('User ID not found in Instagram getUser API response');
+            throw new NotFoundError('User ID not found');
+        })
+        .then((result): string => {
+            if (!result || typeof result !== 'string') {
+                throw new TypeError('Invalid user ID type');
             }
-        }
-
-        return userId;
-    });
+            return result;
+        });
 
 const hasMedia = (post) => post.image_versions2 || post.carousel_media || post.video_versions;
+
 const buildMedia = (post) => {
     let html = '';
 
@@ -99,25 +93,13 @@ const buildMedia = (post) => {
         for (const media of post.carousel_media) {
             const firstImage = media.image_versions2?.candidates[0];
             const firstVideo = media.video_versions?.[0];
-            if (firstVideo) {
-                html += `<video controls autoplay loop poster="${firstImage.url}">`;
-                html += `<source src="${firstVideo.url}"/>`;
-                html += '</video>';
-            } else {
-                html += `<img src="${firstImage.url}"/>`;
-            }
+            html += firstVideo ? `<video controls autoplay loop poster="${firstImage.url}"><source src="${firstVideo.url}"/></video>` : `<img src="${firstImage.url}"/>`;
         }
     } else {
         const mainImage = post.image_versions2?.candidates?.[0];
         const mainVideo = post.video_versions?.[0];
         if (mainImage) {
-            if (mainVideo) {
-                html += `<video controls autoplay loop poster="${mainImage.url}">`;
-                html += `<source src="${mainVideo.url}"/>`;
-                html += '</video>';
-            } else {
-                html += `<img src="${mainImage.url}"/>`;
-            }
+            html += mainVideo ? `<video controls autoplay loop poster="${mainImage.url}"><source src="${mainVideo.url}"/></video>` : `<img src="${mainImage.url}"/>`;
         }
     }
 
@@ -181,4 +163,4 @@ const buildContent = (item, options) => {
     return { title, description };
 };
 
-export { apiUrl, profileUrl, threadUrl, THREADS_QUERY, REPLIES_QUERY, USER_AGENT, extractTokens, getUserId, makeHeader, hasMedia, buildMedia, buildContent };
+export { profileUrl, threadUrl, extractTokens, getUserId, buildContent };

package/lib/routes/tmtpost/column.ts

@@ -0,0 +1,298 @@
+import { type Data, type Route, ViewType } from '@/types';
+
+import { type Context } from 'hono';
+
+import { baseUrl, apiBaseUrl, processItems } from './util';
+
+export const handler = async (ctx: Context): Promise<Data> => {
+    const { id } = ctx.req.param();
+    const limit: number = Number.parseInt(ctx.req.query('limit') ?? '30', 10);
+
+    const targetUrl: string = new URL(`column/${id}`, baseUrl).href;
+    const listApiUrl: string = new URL('v1/categories/multi_content/list', apiBaseUrl).href;
+
+    const query = {
+        subtype: 'post',
+        category_guid: id,
+    };
+
+    return await processItems(limit, query, listApiUrl, targetUrl);
+};
+
+export const route: Route = {
+    path: '/column/:id',
+    name: '最新',
+    url: 'www.tmtpost.com',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/tmtpost/column/6916385',
+    parameters: {
+        id: {
+            description: '专栏 id，可在对应专栏页 URL 中找到',
+            options: [
+                {
+                    label: 'AGI',
+                    value: '6916385',
+                },
+                {
+                    label: '出海',
+                    value: '6998081',
+                },
+                {
+                    label: '创新场景',
+                    value: '3882035',
+                },
+                {
+                    label: '钛度号',
+                    value: '6100587',
+                },
+                {
+                    label: '深度',
+                    value: '3189960',
+                },
+                {
+                    label: '焦点',
+                    value: '6043895',
+                },
+                {
+                    label: '创投',
+                    value: '5994956',
+                },
+                {
+                    label: '汽车',
+                    value: '2573550',
+                },
+                {
+                    label: '3C',
+                    value: '3615534',
+                },
+                {
+                    label: '消费',
+                    value: '3882530',
+                },
+                {
+                    label: '大健康',
+                    value: '3882507',
+                },
+                {
+                    label: '金融',
+                    value: '3882486',
+                },
+                {
+                    label: '钛智宏观',
+                    value: '4277188',
+                },
+                {
+                    label: '产业研究',
+                    value: '5506730',
+                },
+                {
+                    label: '地产',
+                    value: '3882499',
+                },
+                {
+                    label: '大公司',
+                    value: '2446153',
+                },
+                {
+                    label: 'IPO',
+                    value: '6043750',
+                },
+                {
+                    label: '钛度图闻',
+                    value: '5750087',
+                },
+                {
+                    label: '城视',
+                    value: '6998636',
+                },
+                {
+                    label: '创业家',
+                    value: '4273329',
+                },
+                {
+                    label: '人文',
+                    value: '6252390',
+                },
+                {
+                    label: '新职业研究所',
+                    value: '5750104',
+                },
+                {
+                    label: '科普',
+                    value: '5714992',
+                },
+                {
+                    label: '文娱',
+                    value: '2446157',
+                },
+            ],
+        },
+    },
+    description: `:::tip
+若订阅 [AGI](https://www.tmtpost.com/column/6916385)，网址为 \`https://www.tmtpost.com/column/6916385\`，请截取 \`https://www.tmtpost.com/column\` 到末尾的部分 \`6916385\` 作为 \`id\` 参数填入，此时目标路由为 [\`/tmtpost/column/6916385\`](https://rsshub.app/tmtpost/column/6916385)。
+:::
+
+<details>
+  <summary>更多分类</summary>
+
+  | [AGI](https://www.tmtpost.com/column/6916385)        | [出海](https://www.tmtpost.com/column/6998081)       | [创新场景](https://www.tmtpost.com/column/3882035)   | [钛度号](https://www.tmtpost.com/column/6100587)     | [深度](https://www.tmtpost.com/column/3189960)       |
+  | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- |
+  | [6916385](https://rsshub.app/tmtpost/column/6916385) | [6998081](https://rsshub.app/tmtpost/column/6998081) | [3882035](https://rsshub.app/tmtpost/column/3882035) | [6100587](https://rsshub.app/tmtpost/column/6100587) | [3189960](https://rsshub.app/tmtpost/column/3189960) |
+
+  | [焦点](https://www.tmtpost.com/column/6043895)       | [创投](https://www.tmtpost.com/column/5994956)       | [汽车](https://www.tmtpost.com/column/2573550)       | [3C](https://www.tmtpost.com/column/3615534)         | [消费](https://www.tmtpost.com/column/3882530)       |
+  | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- |
+  | [6043895](https://rsshub.app/tmtpost/column/6043895) | [5994956](https://rsshub.app/tmtpost/column/5994956) | [2573550](https://rsshub.app/tmtpost/column/2573550) | [3615534](https://rsshub.app/tmtpost/column/3615534) | [3882530](https://rsshub.app/tmtpost/column/3882530) |
+
+  | [大健康](https://www.tmtpost.com/column/3882507)     | [金融](https://www.tmtpost.com/column/3882486)       | [钛智宏观](https://www.tmtpost.com/column/4277188)   | [产业研究](https://www.tmtpost.com/column/5506730)   | [地产](https://www.tmtpost.com/column/3882499)       |
+  | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- |
+  | [3882507](https://rsshub.app/tmtpost/column/3882507) | [3882486](https://rsshub.app/tmtpost/column/3882486) | [4277188](https://rsshub.app/tmtpost/column/4277188) | [5506730](https://rsshub.app/tmtpost/column/5506730) | [3882499](https://rsshub.app/tmtpost/column/3882499) |
+
+  | [大公司](https://www.tmtpost.com/column/2446153)     | [IPO](https://www.tmtpost.com/column/6043750)        | [钛度图闻](https://www.tmtpost.com/column/5750087)   | [城视](https://www.tmtpost.com/column/6998636)       | [创业家](https://www.tmtpost.com/column/4273329)     |
+  | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- | ---------------------------------------------------- |
+  | [2446153](https://rsshub.app/tmtpost/column/2446153) | [6043750](https://rsshub.app/tmtpost/column/6043750) | [5750087](https://rsshub.app/tmtpost/column/5750087) | [6998636](https://rsshub.app/tmtpost/column/6998636) | [4273329](https://rsshub.app/tmtpost/column/4273329) |
+
+  | [人文](https://www.tmtpost.com/column/6252390)       | [新职业研究所](https://www.tmtpost.com/column/5750104) | [科普](https://www.tmtpost.com/column/5714992)       | [文娱](https://www.tmtpost.com/column/2446157)       |
+  | ---------------------------------------------------- | ------------------------------------------------------ | ---------------------------------------------------- | ---------------------------------------------------- |
+  | [6252390](https://rsshub.app/tmtpost/column/6252390) | [5750104](https://rsshub.app/tmtpost/column/5750104)   | [5714992](https://rsshub.app/tmtpost/column/5714992) | [2446157](https://rsshub.app/tmtpost/column/2446157) |
+
+</details>
+`,
+    categories: ['new-media'],
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['www.tmtpost.com/column/:id'],
+            target: '/column/:id',
+        },
+        {
+            title: 'AGI',
+            source: ['www.tmtpost.com/column/6916385'],
+            target: '/column/6916385',
+        },
+        {
+            title: '出海',
+            source: ['www.tmtpost.com/column/6998081'],
+            target: '/column/6998081',
+        },
+        {
+            title: '创新场景',
+            source: ['www.tmtpost.com/column/3882035'],
+            target: '/column/3882035',
+        },
+        {
+            title: '钛度号',
+            source: ['www.tmtpost.com/column/6100587'],
+            target: '/column/6100587',
+        },
+        {
+            title: '深度',
+            source: ['www.tmtpost.com/column/3189960'],
+            target: '/column/3189960',
+        },
+        {
+            title: '焦点',
+            source: ['www.tmtpost.com/column/6043895'],
+            target: '/column/6043895',
+        },
+        {
+            title: '创投',
+            source: ['www.tmtpost.com/column/5994956'],
+            target: '/column/5994956',
+        },
+        {
+            title: '汽车',
+            source: ['www.tmtpost.com/column/2573550'],
+            target: '/column/2573550',
+        },
+        {
+            title: '3C',
+            source: ['www.tmtpost.com/column/3615534'],
+            target: '/column/3615534',
+        },
+        {
+            title: '消费',
+            source: ['www.tmtpost.com/column/3882530'],
+            target: '/column/3882530',
+        },
+        {
+            title: '大健康',
+            source: ['www.tmtpost.com/column/3882507'],
+            target: '/column/3882507',
+        },
+        {
+            title: '金融',
+            source: ['www.tmtpost.com/column/3882486'],
+            target: '/column/3882486',
+        },
+        {
+            title: '钛智宏观',
+            source: ['www.tmtpost.com/column/4277188'],
+            target: '/column/4277188',
+        },
+        {
+            title: '产业研究',
+            source: ['www.tmtpost.com/column/5506730'],
+            target: '/column/5506730',
+        },
+        {
+            title: '地产',
+            source: ['www.tmtpost.com/column/3882499'],
+            target: '/column/3882499',
+        },
+        {
+            title: '大公司',
+            source: ['www.tmtpost.com/column/2446153'],
+            target: '/column/2446153',
+        },
+        {
+            title: 'IPO',
+            source: ['www.tmtpost.com/column/6043750'],
+            target: '/column/6043750',
+        },
+        {
+            title: '钛度图闻',
+            source: ['www.tmtpost.com/column/5750087'],
+            target: '/column/5750087',
+        },
+        {
+            title: '城视',
+            source: ['www.tmtpost.com/column/6998636'],
+            target: '/column/6998636',
+        },
+        {
+            title: '创业家',
+            source: ['www.tmtpost.com/column/4273329'],
+            target: '/column/4273329',
+        },
+        {
+            title: '人文',
+            source: ['www.tmtpost.com/column/6252390'],
+            target: '/column/6252390',
+        },
+        {
+            title: '新职业研究所',
+            source: ['www.tmtpost.com/column/5750104'],
+            target: '/column/5750104',
+        },
+        {
+            title: '科普',
+            source: ['www.tmtpost.com/column/5714992'],
+            target: '/column/5714992',
+        },
+        {
+            title: '文娱',
+            source: ['www.tmtpost.com/column/2446157'],
+            target: '/column/2446157',
+        },
+    ],
+    view: ViewType.Articles,
+};