rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f812e91

package/lib/v2/showstart/service.js

@@ -0,0 +1,166 @@
+const { HOST } = require('./const');
+const { getAccessToken, post, sortBy, uniqBy } = require('./utils');
+
+async function fetchActivityList(
+    params = {
+        pageNo: '1',
+        pageSize: '30',
+        cityCode: '',
+        activityIds: '',
+        coupon: '',
+        keyword: '',
+        organizerId: '',
+        performerId: '',
+        showStyle: '',
+        showTime: '',
+        showType: '',
+        siteId: '',
+        sortType: '',
+        themeId: '',
+        timeRange: '',
+        tourId: '',
+        type: '',
+        tag: '',
+    }
+) {
+    const accessToken = await getAccessToken();
+    const resp = await post('/web/activity/list', accessToken, params);
+    return resp.result.result.map((item) => formatActivity(item));
+}
+
+function formatActivity(item) {
+    const image = (src) => (src ? `<img src="${src}" />` : '');
+    const time = (time) => (time ? `<p>演出时间：${time}</p>` : '');
+    const address = (cityName, siteName) => (cityName || siteName ? `<p>地址：${[cityName, siteName].join(' - ')}</p>` : '');
+    const performers = (name) => (name ? `<p>艺人：${name}</p>` : '');
+    const price = (price) => (price ? `<p>票价：${price}</p>` : '');
+    return {
+        title: item.title,
+        link: `${HOST}/event/${item.id}`,
+        description: [image(item.poster), time(item.showTime), address(item.cityName, item.siteName), performers(item.performers), price(item.price)].join(''),
+    };
+}
+
+async function fetchPerformerList(
+    params = {
+        pageNo: '1',
+        pageSize: '30',
+        searchKeyword: '',
+        styleId: '',
+    }
+) {
+    const accessToken = await getAccessToken();
+    const resp = await post('/web/performer/list', accessToken, params);
+    return resp.result.result.map((item) => ({
+        title: item.name,
+        link: `${HOST}/artist/${item.id}`,
+        description: `id: ${item.id}`,
+    }));
+}
+
+async function fetchPerformerInfo(
+    params = {
+        performerId: '',
+    }
+) {
+    const accessToken = await getAccessToken();
+    const resp = await post('/web/performer/info', accessToken, params);
+    return {
+        id: params.id,
+        name: resp.result.name,
+        content: resp.result.content,
+        avatar: resp.result.avatar,
+        poster: resp.result.poster,
+        styles: resp.result.styles,
+        activityList: resp.result.activities.map((item) => formatActivity(item)),
+    };
+}
+
+async function fetchBrandInfo(
+    params = {
+        brandId: '',
+    }
+) {
+    const accessToken = await getAccessToken();
+    const resp = await post('/web/brand/info', accessToken, params);
+    return {
+        id: params.id,
+        name: resp.result.name,
+        content: resp.result.content,
+        avatar: resp.result.avatar,
+        poster: resp.result.poster,
+        activityList: resp.result.activities.map((item) => formatActivity(item)),
+    };
+}
+
+async function fetchBrandList(
+    params = {
+        pageNo: '1',
+        pageSize: '30',
+        searchKeyword: '',
+    }
+) {
+    const accessToken = await getAccessToken();
+    const resp = await post('/web/brand/list', accessToken, params);
+    return resp.result.result.map((item) => ({
+        title: item.name,
+        link: `${HOST}/host/${item.id}`,
+        description: `id: ${item.id}`,
+    }));
+}
+
+async function fetchParams() {
+    const accessToken = await getAccessToken();
+    return post('/web/activity/list/params', accessToken);
+}
+
+async function fetchCityList(keyword = '') {
+    const resp = await fetchParams();
+    const cities = sortBy(resp.result, 'cityCode');
+    return cities
+        .filter((item) => item.cityName.includes(keyword.trim()))
+        .map((item) => ({
+            title: item.cityName,
+            link: `${HOST}/event/list?cityCode=${item.cityCode}`,
+            description: `cityCode: ${item.cityCode}`,
+        }));
+}
+
+// styles is embed in each city item
+// so we need to fetch all city items and then extract styles from them
+async function fetchStyleList(keyword = '') {
+    const resp = await fetchParams();
+    let styles = resp.result.flatMap((item) => item.styles);
+    styles = uniqBy(styles, 'key');
+    styles = sortBy(styles, 'key');
+    return styles
+        .filter((item) => item.showName.includes(keyword.trim()))
+        .map((item) => ({
+            title: item.showName,
+            link: `${HOST}/event/list?showStyle=${item.key}`,
+            description: `showStyle: ${item.key}`,
+        }));
+}
+
+async function fetchDictionary(cityCode, showStyle) {
+    const resp = await fetchParams();
+    const target = resp.result.find((item) => item.cityCode === cityCode);
+    if (!target) {
+        return {};
+    }
+    return {
+        cityName: target.cityName,
+        showName: target.styles.find((item) => item.key === showStyle)?.showName,
+    };
+}
+
+module.exports = {
+    fetchActivityList,
+    fetchCityList,
+    fetchStyleList,
+    fetchPerformerList,
+    fetchPerformerInfo,
+    fetchBrandList,
+    fetchBrandInfo,
+    fetchDictionary,
+};

package/lib/v2/showstart/utils.js

@@ -0,0 +1,92 @@
+const got = require('@/utils/got');
+const md5 = require('@/utils/md5');
+
+const uuid = (length = 20) => {
+    const e = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' + Date.now();
+    const r = [];
+    for (let i = 0; i < length; i++) {
+        r.push(e.charAt(Math.floor(Math.random() * e.length)));
+    }
+    return r.join('');
+};
+
+const cookieMap = new Map([['token', uuid(32).toLowerCase()]]);
+
+const devioceInfo = {
+    vendorName: '',
+    deviceMode: '',
+    deviceName: '',
+    systemName: '',
+    systemVersion: '',
+    cpuMode: ' ', // Note the space
+    cpuCores: '',
+    cpuArch: '',
+    memerySize: '',
+    diskSize: '',
+    network: '',
+    resolution: '1920*1080',
+    pixelResolution: '',
+};
+
+const getAccessToken = async () => {
+    const { result } = await post('/waf/gettoken');
+    cookieMap.set('accessToken', result.accessToken.access_token);
+    cookieMap.set('idToken', result.idToken.id_token);
+    return cookieMap.get('accessToken');
+};
+
+const post = async (requestPath, accessToken = md5(Date.now().toString()), payload) => {
+    const traceId = uuid(32) + Date.now();
+
+    const { data: response } = await got.post(`https://www.showstart.com/api${requestPath}`, {
+        headers: {
+            cdeviceinfo: encodeURIComponent(JSON.stringify(devioceInfo)),
+            cdeviceno: cookieMap.get('token'),
+            cookie: [...cookieMap.entries()].map(([key, value]) => `${key}=${value}`).join('; '),
+            crpsign: md5(accessToken + /* sign/cusut (empty) + idToken (empty) + userInfo.userId (empty) + */ 'web' + cookieMap.get('token') + (payload ? JSON.stringify(payload) : '') + requestPath + '999web' + traceId),
+            crtraceid: traceId,
+            csappid: 'web',
+            cterminal: 'web',
+            cusat: accessToken,
+            cusid: '',
+            cusit: '',
+            cusname: '',
+            cusut: '',
+            cversion: '999',
+        },
+        json: payload,
+    });
+
+    return response;
+};
+
+function sortBy(items, key) {
+    return items.sort((a, b) => {
+        if (a[key] < b[key]) {
+            return -1;
+        }
+        if (a[key] > b[key]) {
+            return 1;
+        }
+        return 0;
+    });
+}
+
+function uniqBy(items, key) {
+    const set = new Set();
+    return items.filter((item) => {
+        if (set.has(item[key])) {
+            return false;
+        }
+        set.add(item[key]);
+        return true;
+    });
+}
+
+module.exports = {
+    post,
+    getAccessToken,
+    uuid,
+    sortBy,
+    uniqBy,
+};

package/lib/v2/snowpeak/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/us/new-arrivals': ['NavePnow'],
+    '/us/new-arrivals': ['EthanWng97'],
 };

package/lib/v2/sony/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/downloads/:productType/:productId': ['NavePnow'],
+    '/downloads/:productType/:productId': ['EthanWng97'],
 };

package/lib/v2/sse/disclosure.js

@@ -3,7 +3,6 @@ const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const query = ctx.params.query ?? ''; // beginDate=2018-08-18&endDate=2020-09-01&productId=600696
-    const host = 'https://www.sse.com.cn';
     const queries = query.split('&').reduce((acc, cur) => {
         const [key, value] = cur.split('=');
         acc[key] = value;
@@ -30,11 +29,13 @@ module.exports = async (ctx) => {
         },
     });
 
+    const pdfHost = 'https://static.sse.com.cn';
+
     const items = response.data.result.map((item) => ({
         title: item.TITLE,
-        description: `${host}${item.URL}`,
+        description: `${pdfHost}${item.URL}`,
         pubDate: parseDate(item.ADDDATE),
-        link: `${host}${item.URL}`,
+        link: `${pdfHost}${item.URL}`,
         author: item.SECURITY_NAME,
     }));
 

package/lib/v2/sspu/jwc.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { listId } = ctx.params;
+    const baseUrl = 'https://jwc.sspu.edu.cn';
+
+    const { data: response, url: link } = await got(`${baseUrl}/${listId}/list.htm`);
+    const $ = cheerio.load(response);
+
+    const list = $('.news_list .news')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 15)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const title = item.find('.news_title a');
+            return {
+                title: title.attr('title'),
+                link: `${baseUrl}${title.attr('href')}`,
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+
+                item.description = $('.wp_articlecontent').html();
+                item.pubDate = timezone(parseDate($('.arti_update').text(), 'YYYY-MM-DD HH:mm:ss'), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link,
+        item: items,
+    };
+};

package/lib/v2/sspu/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/jwc/:listId': ['TonyRL'],
+};

package/lib/v2/sspu/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'sspu.edu.cn': {
+        _name: '上海第二工业大学',
+        jwc: [
+            {
+                title: '教务处',
+                docs: 'https://docs.rsshub.app/university#shang-hai-di-er-gong-ye-da-xue',
+                source: ['/jwc/:listId/list.htm'],
+                target: '/sspu/jwc/:listId',
+            },
+        ],
+    },
+};

package/lib/v2/sspu/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/jwc/:listId', require('./jwc'));
+};

package/lib/v2/techcrunch/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/news': ['NavePnow'],
+    '/news': ['EthanWng97'],
 };

package/lib/v2/theatlantic/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/:category': ['NavePnow'],
+    '/:category': ['EthanWng97'],
 };

package/lib/v2/thenewslens/radar.js

@@ -44,12 +44,6 @@ module.exports = {
                 source: ['/review/:sort?', '/'],
                 target: '/thenewslens/review/:sort?',
             },
-            {
-                title: '影音',
-                docs: 'https://docs.rsshub.app/routes/new-media#the-news-lens-guan-jian-ping-lun',
-                source: ['/videos/Projects/:sort?', '/'],
-                target: '/thenewslens/videos/Projects/:sort?',
-            },
         ],
     },
 };

package/lib/v2/thepaper/radar.js

@@ -18,13 +18,13 @@ module.exports = {
                 title: '频道',
                 docs: 'https://docs.rsshub.app/routes/traditional-media#peng-pai-xin-wen-pin-dao',
                 source: ['/'],
-                target: (params, url) => `/thepaper/channel/${new URL(url).search(/channel_(\d+)/)}`,
+                target: (params, url) => `/thepaper/channel/${url.match(/channel_(\d+)/)?.[1]}`,
             },
             {
                 title: '栏目',
                 docs: 'https://docs.rsshub.app/routes/traditional-media#peng-pai-xin-wen-lie-biao',
                 source: ['/'],
-                target: (params, url) => `/thepaper/list/${new URL(url).search(/list_(\d+)/)}`,
+                target: (params, url) => `/thepaper/list/${url.search(/list_(\d+)/)?.[1]}`,
             },
             {
                 title: '澎湃美数组作品集',

package/lib/v2/thoughtco/index.js

@@ -64,7 +64,7 @@ module.exports = async (ctx) => {
                     new Set(
                         content('meta[name="parsely-tags"]')
                             .prop('content')
-                            .split(/,/)
+                            ?.split(/,/)
                             .map((c) => c.trim())
                     )
                 );

package/lib/v2/threads/index.js

@@ -1,146 +1,6 @@
 const got = require('@/utils/got');
-const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-const dayjs = require('dayjs');
-
-const profileUrl = (user) => `https://www.threads.net/@${user}`;
-const threadUrl = (code) => `https://www.threads.net/t/${code}`;
-const apiUrl = 'https://www.threads.net/api/graphql';
-const THREADS_QUERY = 6232751443445612;
-const REPLIES_QUERY = 6307072669391286;
-const USER_AGENT = 'Barcelona 289.0.0.77.109 Android';
-
-const load = async (url) => {
-    // user id fetching needs puppeteer
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    await page.setRequestInterception(true);
-    page.on('request', (request) => {
-        request.resourceType() === 'document' ? request.continue() : request.abort();
-    });
-    await page.goto(url, {
-        waitUntil: 'domcontentloaded',
-    });
-    const response = await page.content();
-    browser.close();
-
-    return cheerio.load(response);
-};
-
-const extractTokens = async (user, ctx) => {
-    const $ = await load(profileUrl(user));
-    const lsd = $('script:contains("LSD"):first')
-        .html()
-        .match(/"LSD",\[],\{"token":"([a-zA-Z0-9@_-]+)"},/)?.[1];
-
-    // This needs puppeteer
-    const userId = $('script:contains("user_id"):first')
-        .html()
-        .match(/"user_id":"(\d+)"/)?.[1];
-
-    ctx.state.json = { lsd, userId };
-    return { lsd, userId };
-};
-
-const makeHeader = (user, lsd) => ({
-    Accept: 'application/json',
-    Host: 'www.threads.net',
-    Origin: 'https://www.threads.net',
-    Referer: `https://www.threads.net/@${user}`,
-    'User-Agent': USER_AGENT,
-    'X-FB-LSD': lsd,
-    'X-IG-App-ID': '238260118697367',
-});
-
-const hasMedia = (post) => post.image_versions2 || post.carousel_media || post.video_versions;
-const buildMedia = (post) => {
-    let html = '';
-
-    if (!post.carousel_media) {
-        const mainImage = post.image_versions2?.candidates?.[0];
-        const mainVideo = post.video_versions?.[0];
-        if (mainImage) {
-            if (!mainVideo) {
-                html += `<img src="${mainImage.url}"/>`;
-            } else {
-                html += `<video controls autoplay loop poster="${mainImage.url}">`;
-                html += `<source src="${mainVideo.url}"/>`;
-                html += '</video>';
-            }
-        }
-    } else {
-        post.carousel_media.forEach((media) => {
-            const firstImage = media.image_versions2?.candidates[0];
-            const firstVideo = media.video_versions?.[0];
-            if (!firstVideo) {
-                html += `<img src="${firstImage.url}"/>`;
-            } else {
-                html += `<video controls autoplay loop poster="${firstImage.url}">`;
-                html += `<source src="${firstVideo.url}"/>`;
-                html += '</video>';
-            }
-        });
-    }
-
-    return html;
-};
-
-const buildContent = (item, options) => {
-    let title = '';
-    let description = '';
-    const quotedPost = item.post.text_post_app_info?.share_info?.quoted_post;
-    const repostedPost = item.post.text_post_app_info?.share_info?.reposted_post;
-    const isReply = item.post.text_post_app_info?.reply_to_author;
-    const embededPost = quotedPost ?? repostedPost;
-
-    if (options.showAuthorInTitle) {
-        title += `@${item.post.user?.username}: `;
-    }
-
-    if (options.showAuthorInDesc) {
-        description += '<p>';
-        if (options.showAuthorAvatarInDesc) {
-            description += `<img src="${item.post.user?.profile_pic_url}" width="48px" height="48px"> `;
-        }
-        description += `<strong>@${item.post.user?.username}</strong>`;
-        if (embededPost) {
-            description += options.showEmojiForQuotesAndReply ? ' 🔁' : ' quoted';
-        } else if (isReply) {
-            description += options.showEmojiForQuotesAndReply ? ' ↩️' : ' replied';
-        }
-        description += ':</p>';
-    }
-
-    if (item.post.caption?.text) {
-        title += item.post.caption?.text;
-        description += `<p>${item.post.caption?.text}</p>`;
-    }
-
-    if (hasMedia(item.post)) {
-        description += `<p>${buildMedia(item.post)}</p>`;
-    }
-
-    if (embededPost) {
-        if (options.showQuotedInTitle) {
-            title += options.showEmojiForQuotesAndReply ? ' 🔁 ' : ' QT: ';
-            title += `@${embededPost.user?.username}: `;
-            title += `"${embededPost.caption?.text}"`;
-        }
-        description += '<blockquote>';
-        description += `<p>${embededPost.caption?.text}</p>`;
-        if (hasMedia(embededPost)) {
-            description += `<p>${buildMedia(embededPost)}</p>`;
-        }
-        description += '— ';
-        if (options.showQuotedAuthorAvatarInDesc) {
-            description += `<img src="${embededPost.user?.profile_pic_url}" width="24px" height="24px"> `;
-        }
-        description += `@${embededPost.user?.username} — `;
-        description += `<a href="${threadUrl(embededPost.code)}">${dayjs(embededPost.taken_at, 'X').toString()}</a>`;
-        description += '</blockquote>';
-    }
-    return { title, description };
-};
+const { PROFILE_QUERY, REPLIES_QUERY, THREADS_QUERY, apiUrl, threadUrl, profileUrl, extractTokens, makeHeader, buildContent } = require('./utils');
 
 module.exports = async (ctx) => {
     const { user, routeParams } = ctx.params;
@@ -159,9 +19,17 @@ module.exports = async (ctx) => {
         replies: params.get('replies') ?? false,
     };
 
-    const headers = makeHeader(user, lsd);
-    const resp = await got.post(apiUrl, {
-        headers,
+    const { data: profileResponse } = await got.post(apiUrl, {
+        headers: makeHeader(user, lsd),
+        form: {
+            lsd,
+            variables: JSON.stringify({ userID: userId }),
+            doc_id: PROFILE_QUERY,
+        },
+    });
+
+    const { data: threadsResponse, request: threadsRequest } = await got.post(apiUrl, {
+        headers: makeHeader(user, lsd),
         form: {
             lsd,
             variables: JSON.stringify({ userID: userId }),
@@ -169,13 +37,14 @@ module.exports = async (ctx) => {
         },
     });
 
+    ctx.state.json.profileData = profileResponse;
     ctx.state.json.request = {
-        headers: resp.request.options.headers,
-        body: resp.request.options.body,
+        headers: threadsRequest.options.headers,
+        body: threadsRequest.options.body,
     };
 
-    const responseBody = resp.data;
-    const threads = responseBody?.data?.mediaData?.threads || [];
+    const userData = profileResponse?.data?.userData?.user || {};
+    const threads = threadsResponse?.data?.mediaData?.threads || [];
 
     const items = threads.flatMap((thread) =>
         thread.thread_items
@@ -195,9 +64,10 @@ module.exports = async (ctx) => {
     ctx.state.json.items = items;
 
     ctx.state.data = {
-        title: user,
+        title: `${userData.full_name} (@${userData.username}) on Threads`,
         link: profileUrl(user),
-        description: user,
+        image: userData.hd_profile_pic_versions?.sort((a, b) => b.width - a.width)[0].url ?? userData.profile_pic_url,
+        description: userData.biography,
         item: items,
     };
 };