rsshub 1.0.0-master.f961961 → 1.0.0-master.f992838

package/lib/v2/jike/topic.js

@@ -11,7 +11,7 @@ module.exports = async (ctx) => {
     const data = await constructTopicEntry(ctx, topicUrl);
 
     if (data) {
-        ctx.state.data.item = topicDataHanding(data);
+        ctx.state.data.item = topicDataHanding(data, ctx);
         if (id === '553870e8e4b0cafb0a1bef68' || id === '55963702e4b0d84d2c30ce6f') {
             ctx.state.data.item = await Promise.all(
                 ctx.state.data.item.map(async (one) => {

package/lib/v2/jike/user.js

@@ -18,6 +18,17 @@ module.exports = async (ctx) => {
     const raw = $('[type = "application/json"]').html();
     const data = JSON.parse(raw).props.pageProps;
 
+    const getLink = (id, type) => {
+        switch (type) {
+            case 'REPOST':
+                return `https://m.okjike.com/reposts/${id}`;
+            case 'MEDIUM':
+                return `https://www.okjike.com/medium/${id}`;
+            default:
+                return `https://m.okjike.com/originalPosts/${id}`;
+        }
+    };
+
     const items = await Promise.all(
         data.posts.map(async (item) => {
             const typeMap = {
@@ -28,16 +39,6 @@ module.exports = async (ctx) => {
                 PERSONAL_UPDATE: '创建新主题',
             };
 
-            const linkMap = {
-                ORIGINAL_POST: `https://m.okjike.com/originalPosts/${item.id}`,
-                REPOST: `https://m.okjike.com/reposts/${item.id}`,
-                MEDIUM: `https://www.okjike.com/medium/${item.id}`,
-                // OFFICIAL_MESSAGE: `https://m.okjike.com/originalPosts/${item.id}`,
-                // ANSWER: `https://m.okjike.com/answer/${item.id}`,
-                // QUESTION: `https://m.okjike.com/originalPosts/${item.id}`,
-                // PERSONAL_UPDATE: `https://m.okjike.com/topic/${item.id}${item.topic && item.topic.id}`,
-            };
-
             let linkTemplate = '';
             if (item.linkInfo && item.linkInfo.linkUrl) {
                 linkTemplate = `<a href="${item.linkInfo.linkUrl}">${item.linkInfo.title}</a><br>`;
@@ -58,6 +59,7 @@ module.exports = async (ctx) => {
                 content = `${content}<br><br>`;
             }
 
+            let repostContent;
             if (item.type === 'REPOST') {
                 const screenNameTemplate = item.target.user ? `<a href="https://m.okjike.com/users/${item.target.user.username}" target="_blank">@${item.target.user.screenName}</a>` : '';
 
@@ -67,7 +69,7 @@ module.exports = async (ctx) => {
                         repostImgTemplate += `<br><img src="${item.thumbnailUrl}">`;
                     });
 
-                const repostContent = `转发 ${screenNameTemplate}: ${item.target.content}${repostImgTemplate}`.replace(/\r\n|\n|\r/g, '<br>');
+                repostContent = `<div class="rsshub-quote">转发 ${screenNameTemplate}: ${item.target.content}${repostImgTemplate}</div>`.replace(/\r\n|\n|\r/g, '<br>');
                 content = `${content}${repostContent}`;
             }
             // 部分功能未知
@@ -105,7 +107,16 @@ module.exports = async (ctx) => {
                 title: `${typeMap[item.type]}了: ${shortenTitle}`,
                 description: `${content}${linkTemplate}${imgTemplate}`.replace(/(<br>|\s)+$/, ''),
                 pubDate: parseDate(item.createdAt),
-                link: linkMap[item.type],
+                link: getLink(item.id, item.type),
+                _extra: repostContent && {
+                    links: [
+                        {
+                            url: getLink(item.target.id, item.target.type),
+                            content_html: repostContent,
+                            type: 'quote',
+                        },
+                    ],
+                },
             };
 
             if (id === 'wenhao1996' && item.topic.id === '553870e8e4b0cafb0a1bef68') {

package/lib/v2/jike/utils.js

@@ -6,7 +6,7 @@ const config = require('@/config').value;
 const videoAPI = 'https://api.ruguoapp.com/1.0/mediaMeta/play?type=ORIGINAL_POST';
 
 module.exports = {
-    topicDataHanding: (data) =>
+    topicDataHanding: (data, ctx) =>
         data.posts.map((item) => {
             let audioName, videoName, linkName;
 
@@ -39,7 +39,9 @@ module.exports = {
             // 添加内容作者信息
             if (item.user) {
                 author = item.user.screenName;
-                description += `<span>用户昵称：${author} <br> Username：${item.user.username}</span><br>`;
+                if (ctx.params.showUid) {
+                    description += `<span>用户昵称：${author} <br> Username：${item.user.username}</span><br>`;
+                }
             }
 
             if (item.linkInfo) {

package/lib/v2/ke/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/researchResults': ['shaomingbo'],
+};

package/lib/v2/ke/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'research.ke.com': {
+        _name: '贝壳研究院 - 房地产行业研究报告',
+        www: [
+            {
+                title: '研究成果',
+                docs: 'https://docs.rsshub.app/other.html#bei-ke-yan-jiu-yuan',
+                source: ['/researchResults'],
+                target: '/researchResults',
+            },
+        ],
+    },
+};

package/lib/v2/ke/results.js

@@ -0,0 +1,36 @@
+const { parseDate } = require('@/utils/parse-date');
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    const response = await got({
+        method: 'post',
+        url: 'https://research.ke.com/apis/consumer-access/index/contents/page',
+        headers: {
+            Referer: 'https://research.ke.com/ResearchResults',
+        },
+        json: {
+            pageIndex: 1,
+            pageSize: 9,
+        },
+    });
+
+    const { status, statusMessage } = response;
+    if (status !== 200) {
+        throw new Error(statusMessage);
+    }
+
+    const { list } = response.data.data;
+
+    ctx.state.data = {
+        title: '房地产行业研究报告',
+        link: 'https://research.ke.com/ResearchResults',
+        description: '研究成果',
+        item: list.map((item) => ({
+            title: item.title,
+            link: `https://research.ke.com/${item.contentTypeId}/ArticleDetail?id=${item.id}`,
+            author: item.author,
+            description: item.guideReading,
+            pubDate: parseDate(item.publishTime),
+        })),
+    };
+};

package/lib/v2/ke/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/researchResults', require('./results'));
+};

package/lib/v2/layoffs/index.js

@@ -0,0 +1,104 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const ROW_COUNT = 100;
+
+const WEBSITE_URL = 'https://layoffs.fyi';
+const SOURCE_URL = `https://airtable.com/v0.3/view/viwA14Z1pM69YIsaW/readSharedViewData?
+stringifiedObjectParams=%7B%22shouldUseNestedResponseFormat%22%3Atrue%7D&
+requestId=reqQcci3rVyhkcAyX&accessPolicy=%7B%22allowedActions%22%3A%5B%7B%22
+modelClassName%22%3A%22view%22%2C%22modelIdSelector%22%3A%22viwA14Z1pM69YIsaW%22%2C%22
+action%22%3A%22readSharedViewData%22%7D%2C%7B%22modelClassName%22%3A%22view%22%2C%22
+modelIdSelector%22%3A%22viwA14Z1pM69YIsaW%22%2C%22action%22%3A%22getMetadataForPrinting
+%22%7D%2C%7B%22modelClassName%22%3A%22view%22%2C%22modelIdSelector
+%22%3A%22viwA14Z1pM69YIsaW%22%2C%22action%22%3A%22readSignedAttachmentUrls
+%22%7D%2C%7B%22modelClassName%22%3A%22row%22%2C%22modelIdSelector%22%3A%22
+rows%20*%5BdisplayedInView%3DviwA14Z1pM69YIsaW%5D%22%2C%22action%22%3A%22
+createBoxDocumentSession%22%7D%2C%7B%22modelClassName%22%3A%22row%22%2C%22
+modelIdSelector%22%3A%22rows%20*%5BdisplayedInView%3DviwA14Z1pM69YIsaW%5D%22%2C%22
+action%22%3A%22createDocumentPreviewSession%22%7D%5D%2C%22shareId%22%3A%22shrqYt5kSqMzHV9R5%22%2C%22
+applicationId%22%3A%22app1PaujS9zxVGUZ4%22%2C%22generationNumber%22%3A0%2C%22
+expires%22%3A%222023-02-02T00%3A00%3A00.000Z%22%2C%22
+signature%22%3A%22096478e17705ffdac261da07c5b4c025aff6d8c6d86be027533fd248913798de%22%7D`;
+
+/**
+ * getMapping can convert either an array or an object
+ * of a specific structure into a Map, where the key is
+ * the name and the value is the Id. It will only parse
+ * entries with 'name' and 'id' in it
+ *
+ * Inputs:
+ *   obj = [{name: 'Apple', id: 1}, {name: 'Peach', id: 2}]
+ * Returns:
+ *   [{1: 'Apple', 2: 'Peach'}, {Apple: 1, Peach: 2}]
+ *
+ * Inputs:
+ *   obj = {Apple: {name: 'Apple', id: 1}, Peach: {name: 'Peach', id: 2}}
+ * Returns:
+ *   [{1: 'Apple', 2: 'Peach'}, {Apple: 1, Peach: 2}]
+ */
+const getMappings = function (obj) {
+    const mapping = new Map();
+    const reverseMapping = new Map();
+    for (const key in obj) {
+        if ('name' in obj[key] && 'id' in obj[key]) {
+            reverseMapping.set(obj[key].name, obj[key].id);
+            mapping.set(obj[key].id, obj[key].name);
+        }
+    }
+    return [mapping, reverseMapping];
+};
+
+module.exports = async (ctx) => {
+    const headers = {
+        'x-airtable-application-id': 'app1PaujS9zxVGUZ4',
+        'x-airtable-inter-service-client': 'webClient',
+        'x-requested-with': 'XMLHttpRequest',
+        'x-time-zone': 'America/Los_Angeles',
+    };
+
+    // Get data from data source
+    const response = await got({
+        method: 'get',
+        url: SOURCE_URL,
+        headers,
+    });
+    const data = response.data.data;
+    const table = data.table;
+
+    // Columns are represented by special IDs
+    const columnReverseMapping = getMappings(table.columns)[1];
+
+    const companyColumnId = columnReverseMapping.get('Company');
+    const dateAddedColumnId = columnReverseMapping.get('Date Added');
+    const numOfLaidOffColumnId = columnReverseMapping.get('# Laid Off');
+    const sourceColumnId = columnReverseMapping.get('Source');
+    const countryColumnId = columnReverseMapping.get('Country');
+    const countryMapping = getMappings(table.columns.filter((col) => col.name === 'Country')[0].typeOptions.choices)[0];
+
+    const rows = table.rows.slice(0, ROW_COUNT);
+    ctx.state.data = {
+        title: 'Tech layoff data feed from layoffs.fyi',
+        link: WEBSITE_URL,
+        description: 'This feed gets tech layoff data from layoffs.fyi and display them in a user friendly way',
+        item: rows.map((row) => {
+            const rowContent = row.cellValuesByColumnId;
+
+            const company = rowContent[companyColumnId];
+            const dateAdded = parseDate(rowContent[dateAddedColumnId]);
+            const source = rowContent[sourceColumnId];
+            const numOfLaidOff = rowContent[numOfLaidOffColumnId] || 'some';
+            const country = countryMapping.get(rowContent[countryColumnId]);
+
+            const title = `${company} Layoffs Happening!`;
+            const description = `${company} lays off ${numOfLaidOff} employees in ${country}. For more details, please visit ${source}.`;
+
+            return {
+                title,
+                description, // the article content
+                pubDate: dateAdded, // Data publish date
+                link: source, // Laid off source link
+            };
+        }),
+    };
+};

package/lib/v2/layoffs/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['BrandNewLifeJackie26'],
+};

package/lib/v2/layoffs/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'layoffs.fyi': {
+        _name: 'Layoffs',
+        '.': [
+            {
+                title: 'Data Tracker Feed',
+                docs: 'https://docs.rsshub.app/en/other.html#layoffs-fyi',
+                source: '/',
+                target: '/layoffs',
+            },
+        ],
+    },
+};

package/lib/v2/layoffs/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index.js'));
+};

package/lib/v2/linkedin/cn/index.js

@@ -0,0 +1,13 @@
+const { parseSearchHit, parseJobPosting } = require('./utils');
+
+const siteUrl = 'https://www.linkedin.cn/incareer/jobs/search';
+
+module.exports = async (ctx) => {
+    const { title, jobs } = await parseSearchHit(ctx);
+    const items = await Promise.all(jobs.map((job) => parseJobPosting(job)));
+    ctx.state.data = {
+        title: `领英 - ${title}`,
+        link: siteUrl,
+        item: items,
+    };
+};

package/lib/v2/linkedin/cn/utils.js

@@ -0,0 +1,119 @@
+const crypto = require('crypto');
+const path = require('path');
+const { art } = require('@/utils/render');
+const got = require('@/utils/got');
+
+const apiUrl = 'https://www.linkedin.cn/karpos/api/graphql';
+const searchHitQueryId = 'searchSearchHitsByJob.be362cd720abd0ebf89b4bbc3253047f';
+const jobPostingQueryId = 'jobsJobPostingsById.3b9573e88687a86607ddb74ff013ef50';
+
+const makeHeader = () => {
+    const sessionId = crypto.randomBytes(8).toString('hex').slice(0, 8);
+    const headers = {
+        Accept: '*/*',
+        Cookie: `JSESSIONID="ajax:${sessionId}"`,
+        'Content-Type': 'application/x-www-form-urlencoded',
+        Referer: 'https://www.linkedin.cn/incareer/jobs/search',
+        'csrf-token': `ajax:${sessionId}`,
+        'x-http-method-override': 'GET',
+        'x-restli-protocol-version': '2.0.0',
+    };
+    return headers;
+};
+
+const period = {
+    1: 'r86400',
+    7: 'r604800',
+    30: 'r2592000',
+};
+
+const location = {
+    china: '102890883',
+    shanghai: '102772228',
+    beijing: '103873152',
+};
+
+const makeVariables = (variables) =>
+    '(' +
+    Object.entries(variables)
+        .filter(([, v]) => v)
+        .map(([k, v]) => `${k}:${encodeURIComponent(v)}`)
+        .join(',') +
+    ')';
+
+const makeBody = (body) => {
+    let output = '';
+    Object.entries(body).forEach(([key, value], index) => {
+        output += `${key}=${value}`;
+        if (index < Object.keys(body).length - 1) {
+            output += '&';
+        }
+    });
+    return output;
+};
+
+const ctxToTitle = (ctx) => {
+    const { keywords } = ctx.params;
+    const { geo, remote, location, period, relevant } = ctx.query;
+    const g = location || geo || 'China';
+    const r = remote ? '远程' : '';
+    const p = period ? `近${period}天` : '';
+    const o = relevant ? '相关' : '最新';
+    return `${o}${p}在${g}的${keywords || ''}${r}工作机会`;
+};
+
+const parseSearchHit = async (ctx) => {
+    const variables = {
+        origin: 'jserp',
+        isForRemoteJobsPage: !!ctx.query.remote,
+        isChinaMultiNationalCorporation: false,
+        count: ctx.query.limit || 20,
+        start: '0',
+        geoUrn: `urn:li:ks_geo:${location[ctx.query.location] || ctx.query.geo || location.china}`,
+        keywords: decodeURIComponent(ctx.params.keywords || ''),
+        f_TPR: period[ctx.query.period] || '',
+        sortType: ctx.query.relevant ? '' : 'DATE_DESCENDING',
+    };
+
+    const resp = await got.post(apiUrl, {
+        headers: makeHeader(),
+        body: makeBody({
+            operationName: 'searchSearchHitsByJob',
+            variables: makeVariables(variables),
+            queryId: searchHitQueryId,
+        }),
+    });
+
+    return {
+        jobs: resp.data.data.searchSearchHitsByJob.elements.map((e) => e.target.jobPosting),
+        title: ctxToTitle(ctx),
+    };
+};
+
+const parseJobPosting = async (jobPosting) => {
+    const entityUrn = jobPosting.entityUrn;
+    const variables = {
+        jobPostingUrn: entityUrn,
+    };
+    const resp = await got.post(apiUrl, {
+        headers: makeHeader(),
+        body: makeBody({
+            operationName: 'jobViewPage',
+            variables: makeVariables(variables),
+            queryId: jobPostingQueryId,
+        }),
+    });
+    const job = resp.data.data.jobsJobPostingsById;
+    return {
+        title: `${jobPosting.companyName} 正在找 ${jobPosting.title}`,
+        link: `https://www.linkedin.cn/incareer/jobs/view/${entityUrn.split(':').pop()}`,
+        guid: `linkedincn:${entityUrn}`,
+        description: art(path.join(__dirname, '../templates/cn/posting.art'), job),
+        pubDate: jobPosting.listedAt,
+    };
+};
+
+module.exports = {
+    parseSearchHit,
+    parseJobPosting,
+};

package/lib/v2/linkedin/jobs.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseJobSearch, KEYWORDS_QUERY_KEY, JOB_TYPES, JOB_TYPES_QUERY_KEY, EXP_LEVELS_QUERY_KEY, parseParamsToSearchParams, EXP_LEVELS, parseParamsToString } = require('./utils');
+
+const BASE_URL = 'https://www.linkedin.com/';
+const JOB_SEARCH_PATH = '/jobs-guest/jobs/api/seeMoreJobPostings/search';
+
+module.exports = async (ctx) => {
+    const jobTypesParam = parseParamsToSearchParams(ctx.params.job_types, JOB_TYPES);
+    const expLevelsParam = parseParamsToSearchParams(ctx.params.exp_levels, EXP_LEVELS);
+
+    let url = new URL(JOB_SEARCH_PATH, BASE_URL);
+    url.searchParams.append(KEYWORDS_QUERY_KEY, ctx.params.keywords || '');
+    url.searchParams.append(JOB_TYPES_QUERY_KEY, jobTypesParam); // see JOB_TYPES in utils.js
+    url.searchParams.append(EXP_LEVELS_QUERY_KEY, expLevelsParam); // see EXPERIENCE_LEVELS in utils.js
+    url = url.toString();
+
+    // Parse job search page
+    const response = await got({
+        method: 'get',
+        url,
+    });
+    const jobs = parseJobSearch(response.data);
+
+    const jobTypes = parseParamsToString(ctx.params.job_types, JOB_TYPES);
+    const expLevels = parseParamsToString(ctx.params.exp_levels, EXP_LEVELS);
+    const feedTitle = 'LinkedIn Job Listing' + (jobTypes ? ` | Job Types: ${jobTypes}` : '') + (expLevels ? ` | Experience Levels: ${expLevels}` : '') + (ctx.params.keywords ? ` | Keywords: ${ctx.params.keywords}` : '');
+    ctx.state.data = {
+        title: feedTitle,
+        link: url,
+        description: 'This feed gets LinkedIn job posts',
+        item: jobs.map((job) => {
+            const title = `${job.company} is hiring ${job.title}`;
+            const description = `Title: ${job.title} | Company: ${job.company} | Location: ${job.location} `;
+
+            return {
+                title, // item title
+                description, // job description
+                pubDate: parseDate(job.pubDate), // data publish date
+                link: job.link, // job source link
+            };
+        }),
+    };
+};

package/lib/v2/linkedin/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/cn/jobs/:keywords?': ['bigfei'],
+    '/jobs': ['BrandNewLifeJackie26'],
+};

package/lib/v2/linkedin/models.js

@@ -0,0 +1,15 @@
+class Job {
+    constructor(title, link, company, location, pubDate, recruiter, description) {
+        this.title = title;
+        this.link = link;
+        this.company = company;
+        this.location = location;
+        this.pubDate = pubDate;
+        this.recruiter = recruiter;
+        this.description = description;
+    }
+}
+
+module.exports = {
+    Job,
+};

package/lib/v2/linkedin/radar.js

@@ -0,0 +1,36 @@
+module.exports = {
+    'linkedin.com': {
+        _name: 'LinkedIn',
+        '.': [
+            {
+                title: 'Job Listing',
+                docs: 'https://docs.rsshub.app/en/other.html#linkedin-jobs',
+                source: '/jobs/search/',
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    const parseRoute = (searchParam) => {
+                        if (typeof searchParam !== 'string') {
+                            return 'all';
+                        }
+                        return searchParam.split(',').join('-');
+                    };
+                    return `/linkedin/jobs/${parseRoute(searchParams.get('f_JT'))}/${parseRoute(searchParams.get('f_E'))}/${searchParams.get('keywords') || ''}`;
+                },
+            },
+        ],
+    },
+    'linkedin.cn': {
+        _name: 'LinkedIn 领英中国',
+        '.': [
+            {
+                title: 'Jobs',
+                docs: 'https://docs.rsshub.app/other.html#linkedin-ling-ying-zhong-guo',
+                source: '/incareer/jobs/search',
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    return `/linkedin/cn/jobs/${searchParams.get('keywords') || ''}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/linkedin/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/cn/jobs/:keywords?', require('./cn/index.js'));
+    router.get('/jobs/:job_types/:exp_levels/:keywords?', require('./jobs.js'));
+};

package/lib/v2/linkedin/templates/cn/posting.art

@@ -0,0 +1,25 @@
+<h2>{{ title }}</h2>
+<p>
+    {{ if (applyMethod.instantOffsiteApply)}}
+    <a href='{{applyMethod.instantOffsiteApply.companyApplyUrl}}'>点击申请</a>
+    {{ /if }}
+</p>
+<p>
+    {{ if (applyMethod.basicOffsiteApply)}}
+    <a href='{{applyMethod.basicOffsiteApply.companyApplyUrl}}'>点击申请</a>
+    {{ /if }}
+</p>
+<p>已有{{numApplies}}人申请此职位， {{numViews}}人查看此职位</p>
+{{ if(compensationDescription) }}
+<p>薪资：{{ compensationDescription || 'N/A' }} </p>
+{{ /if }}
+<p>工作地点： {{ geo.defaultLocalizedName }} </p>
+
+{{ if (company) }}
+<h2>公司介绍</h2>
+<p>{{company.name}}</p>
+<p>员工人数：{{company.employeeCount}}</p>
+<p style="white-space: pre-line">{{company.localizedDescription}}</p>
+{{ /if }}
+<h2>职位介绍</h2>
+<p style="white-space: pre-line">{{description.text}}</p>