rsshub 1.0.0-master.fe99193 → 1.0.0-master.fe9c82a

package/lib/v2/kimlaw/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'kimlaw.or.kr': {
+        _name: '韓國海事法學會',
+        '.': [
+            {
+                title: '学术论文',
+                docs: 'https://docs.rsshub.app/study.html#han-guo-hai-shi-fa-xue-hui',
+                source: ['/67', '/'],
+                target: '/kimlaw/thesis',
+            },
+        ],
+    },
+};

package/lib/v2/kimlaw/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/thesis', require('./thesis'));
+};

package/lib/v2/kimlaw/thesis.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const baseUrl = 'https://www.kimlaw.or.kr';
+
+module.exports = async (ctx) => {
+    const link = `${baseUrl}/67`;
+    const { data: response } = await got(link);
+
+    const $ = cheerio.load(response);
+    const list = $('.li_body')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('a.list_text_title');
+            return {
+                title: a.text(),
+                link: `${baseUrl}${a.attr('href')}`,
+                author: item.find('.name').text(),
+                pubDate: timezone(parseDate(item.find('.time').attr('title')), 9),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+
+                item.description = $('.board_txt_area').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${$('.widget_menu_title').text()} - ${$('head title').text()}`,
+        link,
+        image: 'https://cdn.imweb.me/upload/S20210819f9dd86d20e7d7/9aec17c4e98a5.ico',
+        item: items,
+    };
+};

package/lib/v2/lang/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/live/room/:id': ['MittWillson'],
+};

package/lib/v2/lang/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'lang.live': {
+        _name: '浪 Play 直播',
+        '.': [
+            {
+                title: '直播间开播',
+                docs: 'https://docs.rsshub.app/live.html#lang-play-yuan-zhi-bo',
+                source: ['/room/:id'],
+                target: '/lang/live/room/:id',
+            },
+        ],
+    },
+};

package/lib/v2/lang/room.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+    const url = `https://www.lang.live/room/${id}`;
+
+    const api = 'https://api.lang.live/langweb/v1/room/liveinfo';
+    const {
+        data: { data },
+    } = await got(api, {
+        searchParams: {
+            room_id: id,
+        },
+    });
+
+    let item = [];
+    const name = data.live_info.nickname;
+    if (data.live_info.live_status === 1) {
+        item = [
+            {
+                title: `${name} 开播了`,
+                link: url,
+                guid: `lang:live:room:${id}:${data.live_info.live_id}`,
+                description: art(path.join(__dirname, 'templates/room.art'), {
+                    live_info: data.live_info,
+                }),
+            },
+        ];
+    }
+
+    ctx.state.data = {
+        title: `${name} 的浪 Play 直播`,
+        description: data.live_info.sign,
+        link: url,
+        item,
+        allowEmpty: true,
+    };
+};

package/lib/v2/lang/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/live/room/:id', require('./room'));
+};

package/lib/v2/lang/templates/room.art

@@ -0,0 +1 @@
+<img src="{{ live_info.headimg }}">

package/lib/v2/layoffs/index.js

@@ -0,0 +1,134 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const ROW_COUNT = 100;
+
+const WEBSITE_URL = 'https://layoffs.fyi';
+const ENTRY_URL = 'https://airtable.com/embed/shrqYt5kSqMzHV9R5/tbl8c8kanuNB6bPYr';
+const AIRTABLE_HOST = 'https://airtable.com';
+
+/**
+ * getMapping can convert either an array or an object
+ * of a specific structure into a Map, where the key is
+ * the name and the value is the Id. It will only parse
+ * entries with 'name' and 'id' in it
+ *
+ * Inputs:
+ *   obj = [{name: 'Apple', id: 1}, {name: 'Peach', id: 2}]
+ * Returns:
+ *   [{1: 'Apple', 2: 'Peach'}, {Apple: 1, Peach: 2}]
+ *
+ * Inputs:
+ *   obj = {Apple: {name: 'Apple', id: 1}, Peach: {name: 'Peach', id: 2}}
+ * Returns:
+ *   [{1: 'Apple', 2: 'Peach'}, {Apple: 1, Peach: 2}]
+ */
+const getMappings = function (obj) {
+    const mapping = new Map();
+    const reverseMapping = new Map();
+    for (const key in obj) {
+        if ('name' in obj[key] && 'id' in obj[key]) {
+            reverseMapping.set(obj[key].name, obj[key].id);
+            mapping.set(obj[key].id, obj[key].name);
+        }
+    }
+    return [mapping, reverseMapping];
+};
+
+module.exports = async (ctx) => {
+    const headers = {
+        'x-airtable-application-id': 'app1PaujS9zxVGUZ4',
+        'x-airtable-inter-service-client': 'webClient',
+        'x-requested-with': 'XMLHttpRequest',
+        'x-time-zone': 'America/Los_Angeles',
+    };
+
+    // Get the latest data source url
+    let dataSourceUrl = await ctx.cache.get(ENTRY_URL);
+    let cacheInvalid = false;
+    let response;
+    if (!dataSourceUrl) {
+        cacheInvalid = true;
+    } else {
+        // Try to fetch data,
+        // it may fail due to outdated url
+        try {
+            response = await got({
+                method: 'get',
+                url: dataSourceUrl,
+                headers,
+            });
+            if (response.statusCode >= 400) {
+                cacheInvalid = true;
+            }
+        } catch {
+            cacheInvalid = true;
+        }
+    }
+
+    if (cacheInvalid) {
+        // Refetch the data source link from entry page
+        const sourcePage = await got({
+            method: 'get',
+            url: ENTRY_URL,
+        });
+        const $ = cheerio.load(sourcePage.data);
+        dataSourceUrl =
+            AIRTABLE_HOST +
+            $('script')
+                .text()
+                .match(/urlWithParams: "(.*?)"/)[1]
+                .replace(/\\u002F/g, '/');
+
+        // Cache it again
+        ctx.cache.set(ENTRY_URL, dataSourceUrl);
+
+        // Refetch the data source
+        response = await got({
+            method: 'get',
+            url: dataSourceUrl,
+            headers,
+        });
+    }
+
+    // Get data from data source
+    const data = response.data.data;
+    const table = data.table;
+
+    // Columns are represented by special IDs
+    const columnReverseMapping = getMappings(table.columns)[1];
+
+    const companyColumnId = columnReverseMapping.get('Company');
+    const dateAddedColumnId = columnReverseMapping.get('Date Added');
+    const numOfLaidOffColumnId = columnReverseMapping.get('# Laid Off');
+    const sourceColumnId = columnReverseMapping.get('Source');
+    const countryColumnId = columnReverseMapping.get('Country');
+    const countryMapping = getMappings(table.columns.filter((col) => col.name === 'Country')[0].typeOptions.choices)[0];
+
+    const rows = table.rows.slice(0, ROW_COUNT);
+    ctx.state.data = {
+        title: 'Tech layoff data feed from layoffs.fyi',
+        link: WEBSITE_URL,
+        description: 'This feed gets tech layoff data from layoffs.fyi and display them in a user friendly way',
+        item: rows.map((row) => {
+            const rowContent = row.cellValuesByColumnId;
+
+            const company = rowContent[companyColumnId];
+            const dateAdded = parseDate(rowContent[dateAddedColumnId]);
+            const source = rowContent[sourceColumnId];
+            const numOfLaidOff = rowContent[numOfLaidOffColumnId] || 'some';
+            const country = countryMapping.get(rowContent[countryColumnId]);
+
+            const title = `${company} Layoffs Happening!`;
+            const description = `${company} lays off ${numOfLaidOff} employees in ${country}. For more details, please visit ${source}.`;
+
+            return {
+                title,
+                description, // the article content
+                pubDate: dateAdded, // Data publish date
+                link: source, // Laid off source link
+            };
+        }),
+    };
+};

package/lib/v2/layoffs/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['BrandNewLifeJackie26'],
+};

package/lib/v2/layoffs/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'layoffs.fyi': {
+        _name: 'Layoffs',
+        '.': [
+            {
+                title: 'Data Tracker Feed',
+                docs: 'https://docs.rsshub.app/en/other.html#layoffs-fyi',
+                source: '/',
+                target: '/layoffs',
+            },
+        ],
+    },
+};

package/lib/v2/layoffs/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index.js'));
+};

package/lib/v2/learnblockchain/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/posts/:cid/:sort?': ['running-grass'],
+};

package/lib/v2/learnblockchain/posts.js

@@ -0,0 +1,43 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseRelativeDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const cid = ctx.params.cid || 'all';
+    const sort = ctx.params.sort;
+
+    let url = 'https://learnblockchain.cn/categories/';
+    url += cid + '/';
+
+    if (sort) {
+        url += sort + '/';
+    }
+
+    const response = await got(url);
+
+    const data = response.data;
+    const $ = cheerio.load(data);
+    const list = $('div.stream-list section.stream-list-item');
+
+    ctx.state.data = {
+        title: `登链社区--${cid}`,
+        link: url,
+        description: `登链社区`,
+        item:
+            list &&
+            list
+                .map((idx, ite) => {
+                    const item = $(ite);
+                    const json = {
+                        title: item.find('h2.title').text().trim(),
+                        description: item.find('div.excerpt').text().trim(),
+                        pubDate: parseRelativeDate(item.find('.author li:nth-child(2)').text().replace('发布于', '').trim()),
+                        link: item.find('h2.title a').attr('href').trim(),
+                        author: item.find('.author li:nth-child(1)').text().trim(),
+                    };
+
+                    return json;
+                })
+                .get(),
+    };
+};

package/lib/v2/learnblockchain/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'learnblockchain.cn': {
+        _name: '登链社区',
+        '.': [
+            {
+                title: '分类文章',
+                docs: 'https://docs.rsshub.app/programming.html#deng-lian-she-qu-wen-zhang',
+                source: '/categories/:cid/:sort?',
+                target: (params) => `/learnblockchain/posts/${params.cid || 'all'}/${params.sort || 'featured'}`,
+            },
+            {
+                title: '全部文章',
+                docs: 'https://docs.rsshub.app/programming.html#deng-lian-she-qu-wen-zhang',
+                source: '*',
+                target: () => '/learnblockchain/posts/all/',
+            },
+        ],
+    },
+};

package/lib/v2/learnblockchain/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/posts/:cid/:sort?', require('./posts'));
+};

package/lib/v2/linkedin/cn/index.js

@@ -0,0 +1,13 @@
+const { parseSearchHit, parseJobPosting } = require('./utils');
+
+const siteUrl = 'https://www.linkedin.cn/incareer/jobs/search';
+
+module.exports = async (ctx) => {
+    const { title, jobs } = await parseSearchHit(ctx);
+    const items = await Promise.all(jobs.map((job) => parseJobPosting(ctx, job)));
+    ctx.state.data = {
+        title: `领英 - ${title}`,
+        link: siteUrl,
+        item: items,
+    };
+};

package/lib/v2/linkedin/cn/renderer.js

@@ -0,0 +1,179 @@
+const text_tag = {
+    LINE_BREAK: 0,
+    INLINE_CODE: 1,
+    LEGACY_PUBLISHING_EMPHASIS: 2,
+    LIST_ITEM: 3,
+    LIST: 4,
+    STRIKETHROUGH: 5,
+    SUBSCRIPT: 6,
+    SUPERSCRIPT: 7,
+    UNDERLINE: 8,
+    BOLD: 9,
+    ITALIC: 10,
+    PARAGRAPH: 11,
+};
+
+class TreeNode {
+    constructor(attr) {
+        this.attr = attr;
+        this.start = attr.start;
+        this.end = attr.start + attr.length;
+        this.children = [];
+    }
+}
+class Ucs2Text {
+    constructor(text) {
+        if (Array.isArray(text)) {
+            this.string = null;
+            this.codePoints = text;
+        } else {
+            this.string = text;
+            const k = text.length;
+            const d = [];
+
+            let l,
+                e,
+                m = 0;
+            while (m < k) {
+                l = text.charCodeAt(m++);
+                if (l >= 55296 && l <= 56319 && m < k) {
+                    e = text.charCodeAt(m++);
+                    if (56320 === (64512 & e)) {
+                        d.push(((1023 & l) << 10) + (1023 & e) + 65536);
+                    } else {
+                        d.push(l);
+                        m--;
+                    }
+                } else {
+                    d.push(l);
+                }
+            }
+
+            this.codePoints = d;
+        }
+        this.length = this.codePoints.length;
+    }
+    substring(start, end) {
+        let _len = this.length,
+            _start = start,
+            _end = end;
+        if (_end === 0) {
+            return new Ucs2Text('');
+        }
+        if (((isNaN(_start) || 0 > _start) && (_start = 0), (isNaN(_end) || 0 > _end) && (_end = _len), _start > _len && (_start = _len), _end > _len && (_end = _len), _end < _start)) {
+            _len = [_start, _end];
+            _end = _len[0];
+            _start = _len[1];
+        }
+        return new Ucs2Text(_start === _end ? '' : this.codePoints.slice(_start, _end));
+    }
+    slice(a, b) {
+        return this.substring(a, b).toString();
+    }
+    toString() {
+        return null !== this.string
+            ? this.string
+            : this.codePoints
+                  .map((a) => {
+                      let b = '';
+                      return 65535 < a && ((b += String.fromCharCode((((a -= 65536) >>> 10) & 1023) | 55296)), (a = 1023 & (56320 | a))), b + String.fromCharCode(a);
+                  })
+                  .join('');
+    }
+}
+
+const parseAttr = (description) => {
+    const { attributes, text } = description;
+    const attrs = [...attributes];
+    attrs.sort((b, d) => {
+        const f = b.start + b.length,
+            e = d.start + d.length;
+        return f !== e ? f - e : text_tag[b.detailData.style] - text_tag[d.detailData.style];
+    });
+    const n = [];
+    for (const q of attrs) {
+        const p = new TreeNode(q);
+        const w = q.start;
+        while (0 < n.length && n[n.length - 1].attr.start >= w) {
+            p.children.push(n.pop());
+        }
+        p.children.reverse();
+        n.push(p);
+    }
+
+    const m = new Ucs2Text(text);
+    const renderSingle = (node) => {
+        const style = node?.attr?.detailData?.style;
+        switch (style) {
+            case 'BOLD':
+                return `<b>${node.text}</b>`;
+            case 'LINE_BREAK':
+                return `<br>`;
+            case 'LIST_ITEM':
+                return `<li>${node.text}</li>`;
+            case 'LIST':
+                return `<ul>${node.text}</ul>`;
+            case 'INLINE_CODE':
+                return `<code>${node.text}</code>`;
+            case 'LEGACY_PUBLISHING_EMPHASIS':
+                return `<em>${node.text}</em>`;
+            case 'STRIKETHROUGH':
+                return `<s>${node.text}</s>`;
+            case 'SUBSCRIPT':
+                return `<sub>${node.text}</sub>`;
+            case 'SUPERSCRIPT':
+                return `<sup>${node.text}</sup>`;
+            case 'UNDERLINE':
+                return `<u>${node.text}</u>`;
+            case 'ITALIC':
+                return `<i>${node.text}</i>`;
+            case 'PARAGRAPH':
+                return `<p>${node.text}</p>`;
+            default:
+                return node.text || node;
+        }
+    };
+
+    const render = (node) => {
+        // DFS render
+        if (node.children.length === 0) {
+            node.text = m.slice(node.start, node.end);
+            return renderSingle(node);
+        }
+        const res = [];
+        let s = node.start;
+        for (const child of node.children) {
+            if (s < child.start) {
+                res.push(m.slice(s, child.start)); // added text before 1st child node
+            }
+            s = child.end;
+            res.push(render(child));
+        }
+
+        if (s < node.end) {
+            res.push(m.slice(s, node.end)); // added text after the last child node
+        }
+        node.text = res.join('');
+        return renderSingle(node);
+    };
+
+    const q = [];
+    let p = 0;
+    for (let w = 0; w < n.length; w++) {
+        // BFS render
+        const e = n[w];
+        if (e.start > p) {
+            q.push(m.slice(p, e.start));
+        }
+        p = e.end;
+        q.push(render(e));
+    }
+    if (p < m.length) {
+        q.push(m.slice(p, m.length));
+    }
+    return q.join('');
+};
+
+module.exports = {
+    parseAttr,
+};

package/lib/v2/linkedin/cn/utils.js

@@ -0,0 +1,122 @@
+const crypto = require('crypto');
+const path = require('path');
+const { art } = require('@/utils/render');
+const got = require('@/utils/got');
+const { parseAttr } = require('./renderer');
+
+const apiUrl = 'https://www.linkedin.cn/karpos/api/graphql';
+const searchHitQueryId = 'searchSearchHitsByJob.be362cd720abd0ebf89b4bbc3253047f';
+const jobPostingQueryId = 'jobsJobPostingsById.3b9573e88687a86607ddb74ff013ef50';
+
+const makeHeader = () => {
+    const sessionId = crypto.randomBytes(8).toString('hex').slice(0, 8);
+    const headers = {
+        Accept: '*/*',
+        Cookie: `JSESSIONID="ajax:${sessionId}"`,
+        'Content-Type': 'application/x-www-form-urlencoded',
+        Referer: 'https://www.linkedin.cn/incareer/jobs/search',
+        'csrf-token': `ajax:${sessionId}`,
+        'x-http-method-override': 'GET',
+        'x-restli-protocol-version': '2.0.0',
+    };
+    return headers;
+};
+
+const period = {
+    1: 'r86400',
+    7: 'r604800',
+    30: 'r2592000',
+};
+
+const location = {
+    china: '102890883',
+    shanghai: '102772228',
+    beijing: '103873152',
+};
+
+const makeVariables = (variables) =>
+    '(' +
+    Object.entries(variables)
+        .filter(([, v]) => v)
+        .map(([k, v]) => `${k}:${encodeURIComponent(v)}`)
+        .join(',') +
+    ')';
+
+const makeBody = (body) => {
+    let output = '';
+    Object.entries(body).forEach(([key, value], index) => {
+        output += `${key}=${value}`;
+        if (index < Object.keys(body).length - 1) {
+            output += '&';
+        }
+    });
+    return output;
+};
+
+const ctxToTitle = (ctx) => {
+    const { keywords } = ctx.params;
+    const { geo, remote, location, period, relevant } = ctx.query;
+    const g = location || geo || 'China';
+    const r = remote ? '远程' : '';
+    const p = period ? `近${period}天` : '';
+    const o = relevant ? '相关' : '最新';
+    return `${o}${p}在${g}的${keywords || ''}${r}工作机会`;
+};
+
+const parseSearchHit = async (ctx) => {
+    const variables = {
+        origin: 'jserp',
+        isForRemoteJobsPage: !!ctx.query.remote,
+        isChinaMultiNationalCorporation: false,
+        count: ctx.query.limit || 20,
+        start: '0',
+        geoUrn: `urn:li:ks_geo:${location[ctx.query.location] || ctx.query.geo || location.china}`,
+        keywords: decodeURIComponent(ctx.params.keywords || ''),
+        f_TPR: period[ctx.query.period] || '',
+        sortType: ctx.query.relevant ? '' : 'DATE_DESCENDING',
+    };
+
+    const resp = await got.post(apiUrl, {
+        headers: makeHeader(),
+        body: makeBody({
+            operationName: 'searchSearchHitsByJob',
+            variables: makeVariables(variables),
+            queryId: searchHitQueryId,
+        }),
+    });
+
+    return {
+        jobs: resp.data.data.searchSearchHitsByJob.elements.map((e) => e.target.jobPosting),
+        title: ctxToTitle(ctx),
+    };
+};
+
+const parseJobPosting = (ctx, jobPosting) => {
+    const entityUrn = jobPosting.entityUrn;
+    return ctx.cache.tryGet(`linkedincn:${entityUrn}`, async () => {
+        const resp = await got.post(apiUrl, {
+            headers: makeHeader(),
+            body: makeBody({
+                operationName: 'jobViewPage',
+                variables: makeVariables({
+                    jobPostingUrn: entityUrn,
+                }),
+                queryId: jobPostingQueryId,
+            }),
+        });
+        const job = resp.data.data.jobsJobPostingsById;
+        job.desc = parseAttr(job.description);
+        return {
+            title: `${jobPosting.companyName} 正在找 ${jobPosting.title}`,
+            link: `https://www.linkedin.cn/incareer/jobs/view/${entityUrn.split(':').pop()}`,
+            guid: `linkedincn:${entityUrn}`,
+            description: art(path.join(__dirname, '../templates/cn/posting.art'), job),
+            pubDate: jobPosting.listedAt,
+        };
+    });
+};
+
+module.exports = {
+    parseSearchHit,
+    parseJobPosting,
+};