rsshub 1.0.0-master.fe99193 → 1.0.0-master.fe9c82a

package/lib/v2/linkedin/jobs.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseJobSearch, KEYWORDS_QUERY_KEY, JOB_TYPES, JOB_TYPES_QUERY_KEY, EXP_LEVELS_QUERY_KEY, parseParamsToSearchParams, EXP_LEVELS, parseParamsToString } = require('./utils');
+
+const BASE_URL = 'https://www.linkedin.com/';
+const JOB_SEARCH_PATH = '/jobs-guest/jobs/api/seeMoreJobPostings/search';
+
+module.exports = async (ctx) => {
+    const jobTypesParam = parseParamsToSearchParams(ctx.params.job_types, JOB_TYPES);
+    const expLevelsParam = parseParamsToSearchParams(ctx.params.exp_levels, EXP_LEVELS);
+
+    let url = new URL(JOB_SEARCH_PATH, BASE_URL);
+    url.searchParams.append(KEYWORDS_QUERY_KEY, ctx.params.keywords || '');
+    url.searchParams.append(JOB_TYPES_QUERY_KEY, jobTypesParam); // see JOB_TYPES in utils.js
+    url.searchParams.append(EXP_LEVELS_QUERY_KEY, expLevelsParam); // see EXPERIENCE_LEVELS in utils.js
+    url = url.toString();
+
+    // Parse job search page
+    const response = await got({
+        method: 'get',
+        url,
+    });
+    const jobs = parseJobSearch(response.data);
+
+    const jobTypes = parseParamsToString(ctx.params.job_types, JOB_TYPES);
+    const expLevels = parseParamsToString(ctx.params.exp_levels, EXP_LEVELS);
+    const feedTitle = 'LinkedIn Job Listing' + (jobTypes ? ` | Job Types: ${jobTypes}` : '') + (expLevels ? ` | Experience Levels: ${expLevels}` : '') + (ctx.params.keywords ? ` | Keywords: ${ctx.params.keywords}` : '');
+    ctx.state.data = {
+        title: feedTitle,
+        link: url,
+        description: 'This feed gets LinkedIn job posts',
+        item: jobs.map((job) => {
+            const title = `${job.company} is hiring ${job.title}`;
+            const description = `Title: ${job.title} | Company: ${job.company} | Location: ${job.location} `;
+
+            return {
+                title, // item title
+                description, // job description
+                pubDate: parseDate(job.pubDate), // data publish date
+                link: job.link, // job source link
+            };
+        }),
+    };
+};

package/lib/v2/linkedin/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/cn/jobs/:keywords?': ['bigfei'],
+    '/jobs': ['BrandNewLifeJackie26'],
+};

package/lib/v2/linkedin/models.js

@@ -0,0 +1,15 @@
+class Job {
+    constructor(title, link, company, location, pubDate, recruiter, description) {
+        this.title = title;
+        this.link = link;
+        this.company = company;
+        this.location = location;
+        this.pubDate = pubDate;
+        this.recruiter = recruiter;
+        this.description = description;
+    }
+}
+
+module.exports = {
+    Job,
+};

package/lib/v2/linkedin/radar.js

@@ -0,0 +1,36 @@
+module.exports = {
+    'linkedin.com': {
+        _name: 'LinkedIn',
+        '.': [
+            {
+                title: 'Job Listing',
+                docs: 'https://docs.rsshub.app/en/other.html#linkedin-jobs',
+                source: '/jobs/search/',
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    const parseRoute = (searchParam) => {
+                        if (typeof searchParam !== 'string') {
+                            return 'all';
+                        }
+                        return searchParam.split(',').join('-');
+                    };
+                    return `/linkedin/jobs/${parseRoute(searchParams.get('f_JT'))}/${parseRoute(searchParams.get('f_E'))}/${searchParams.get('keywords') || ''}`;
+                },
+            },
+        ],
+    },
+    'linkedin.cn': {
+        _name: 'LinkedIn 领英中国',
+        '.': [
+            {
+                title: 'Jobs',
+                docs: 'https://docs.rsshub.app/other.html#linkedin-ling-ying-zhong-guo',
+                source: '/incareer/jobs/search',
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    return `/linkedin/cn/jobs/${searchParams.get('keywords') || ''}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/linkedin/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/cn/jobs/:keywords?', require('./cn/index.js'));
+    router.get('/jobs/:job_types/:exp_levels/:keywords?', require('./jobs.js'));
+};

package/lib/v2/linkedin/templates/cn/posting.art

@@ -0,0 +1,25 @@
+<h2>{{ title }}</h2>
+<p>
+    {{ if (applyMethod.instantOffsiteApply)}}
+    <a href='{{applyMethod.instantOffsiteApply.companyApplyUrl}}'>点击申请</a>
+    {{ /if }}
+</p>
+<p>
+    {{ if (applyMethod.basicOffsiteApply)}}
+    <a href='{{applyMethod.basicOffsiteApply.companyApplyUrl}}'>点击申请</a>
+    {{ /if }}
+</p>
+<p>已有{{numApplies}}人申请此职位， {{numViews}}人查看此职位</p>
+{{ if(compensationDescription) }}
+<p>薪资：{{ compensationDescription || 'N/A' }} </p>
+{{ /if }}
+<p>工作地点： {{ geo.defaultLocalizedName }} </p>
+
+{{ if (company) }}
+<h2>公司介绍</h2>
+<p>{{company.name}}</p>
+<p>员工人数：{{company.employeeCount}}</p>
+<p>{{company.localizedDescription}}</p>
+{{ /if }}
+<h2>职位介绍</h2>
+<article>{{@ desc}}</article>

package/lib/v2/linkedin/utils.js

@@ -0,0 +1,121 @@
+const cheerio = require('cheerio');
+const { Job } = require('./models');
+
+/**
+ * Constants
+ */
+const KEYWORDS_QUERY_KEY = 'keywords';
+
+const JOB_TYPES_QUERY_KEY = 'f_JT';
+
+const JOB_TYPES = {
+    F: 'full-time',
+    P: 'part-time',
+    C: 'contract',
+};
+
+const EXP_LEVELS_QUERY_KEY = 'f_E';
+
+const EXP_LEVELS = {
+    1: 'internship',
+    2: 'entry',
+    3: 'associate',
+    4: 'mid-senior',
+    5: 'director',
+};
+
+/**
+ * Params parsing
+ */
+
+/**
+ * Parse params in route into valid query params
+ * e.g. /jobs/:keywords/:job_types/:exp_levels
+ *   if job_types is 'C-T'
+ *   the output search param is 'C,T', which can be
+ *   used as search param in query
+ * @param {String} params params in route (separated by '-')
+ * @param {Object} map valid value map for this param
+ * @returns search params separated by ',' that can be used
+ *   as search param in url
+ */
+function parseParamsToSearchParams(params, map) {
+    const validParamValues = params.split('-').filter((v) => v in map);
+    return validParamValues.join(',');
+}
+
+/**
+ * Parse params in route into human readable strings
+ * e.g. /jobs/:keywords/:job_types/:exp_levels
+ *   if job_types is 'C-T'
+ *   the output is 'contract,full-time'
+ * @param {String} params params in route (separated by '-')
+ * @param {Object} map valid value map for this param
+ * @returns param value strings separated by ','
+ */
+function parseParamsToString(params, map) {
+    const validParamValues = params
+        .split('-')
+        .filter((v) => v in map)
+        .map((v) => map[v]);
+    return validParamValues.join(',');
+}
+
+/**
+ * HTML page parsing
+ */
+
+/**
+ * Parse job search page
+ * Example page: https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?keywords=Software%20Engineer&location=United%20States&locationId=&geoId=103644278&sortBy=R&f_TPR=&position=1&pageNum=0
+ *
+ * @param {String} data HTML string of job search page
+ * @returns {Job[]} Array of jobs with data filled
+ */
+function parseJobSearch(data) {
+    const $ = cheerio.load(data);
+
+    // Parse data
+    const jobs = $('li')
+        .map((i, elem) => {
+            const elemHtml = $(elem);
+            const link = elemHtml.find('a.base-card__full-link').attr('href').split('?')[0];
+            const title = elemHtml.find('h3.base-search-card__title').text().trim();
+            const company = elemHtml.find('h4.base-search-card__subtitle').text().trim();
+            const location = elemHtml.find('span.job-search-card__location').text().trim();
+            const pubDate = elemHtml.find('time').attr('datetime');
+
+            return new Job(title, link, company, location, pubDate);
+        })
+        .toArray();
+    return jobs;
+}
+
+/**
+ * Parse job detail page
+ * Example page: https://www.linkedin.com/jobs/view/software-engineer-backend-junior-at-genies-3429649821?trk=public_jobs_topcard-title
+ *
+ * @param {String} data HTML string of job detail page
+ * @returns {Job} Job details
+ */
+function parseJobDetail(data) {
+    const job = new Job();
+    const $ = cheerio.load(data);
+
+    job.recruiter = $('a.message-the-recruiter__cta').attr(`href`);
+    job.description = $('div.description__text description__text--rich').text();
+
+    return job;
+}
+
+module.exports = {
+    parseParamsToSearchParams,
+    parseParamsToString,
+    parseJobDetail,
+    parseJobSearch,
+    JOB_TYPES,
+    JOB_TYPES_QUERY_KEY,
+    EXP_LEVELS,
+    EXP_LEVELS_QUERY_KEY,
+    KEYWORDS_QUERY_KEY,
+};

package/lib/v2/liulinblog/utils.js

@@ -4,12 +4,12 @@ const { parseDate } = require('@/utils/parse-date');
 
 const processList = (res) => {
     const $ = cheerio.load(res.data);
-    const list = $('div.posts-wrapper > div.col-lg-12')
+    const list = $('div.archive.container > div.row > div > div > div.row.posts-wrapper.scroll > div')
         .map((_, item) => ({
             title: $(item).find('h2.entry-title').text(),
             link: $(item).find('h2.entry-title > a').attr('href'),
             pubDate: parseDate($(item).find('time').attr('datetime')),
-            author: $(item).find('span.meta-author > a').attr('title'),
+            author: '60秒读懂世界',
         }))
         .get();
 
@@ -22,8 +22,11 @@ const processItems = async (list, ctx) => {
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got(item.link);
                 const $ = cheerio.load(detailResponse.data);
-
-                item.description = $('div.entry-wrapper').html();
+                const fullContent = $('div > div.entry-wrapper > div.entry-content');
+                fullContent.find('img').remove();
+                fullContent.find('div.pt-0').remove();
+                fullContent.find('div.post-note').remove();
+                item.description = fullContent.html();
 
                 return item;
             })

package/lib/v2/lofter/user.js

@@ -1,9 +1,13 @@
 const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const name = ctx.params.name ?? 'i';
     const limit = ctx.query.limit ? parseInt(ctx.query.limit) : '50';
+    if (!isValidHost(name)) {
+        throw Error('Invalid name');
+    }
 
     const rootUrl = `${name}.lofter.com`;
 
@@ -33,10 +37,10 @@ module.exports = async (ctx) => {
         description:
             JSON.parse(item.post.photoLinks || `[]`)
                 .map((photo) => {
-                    if (photo.raw.match(/\/\/nos\.netease\.com\//)) {
+                    if (photo.raw?.match(/\/\/nos\.netease\.com\//)) {
                         photo.raw = `https://${photo.raw.match(/(imglf\d)/)[0]}.lf127.net${photo.raw.match(/\/\/nos\.netease\.com\/imglf\d(.*)/)[1]}`;
                     }
-                    return `<img src="${photo.raw}">`;
+                    return `<img src="${photo.raw || photo.orign}">`;
                 })
                 .join('') +
             JSON.parse(item.post.embed ? `[${item.post.embed}]` : `[]`)

package/lib/v2/lovelive-anime/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
     '/news/:option?': ['axojhf'],
-    '/topics/:abbr/:category?': ['axojhf'],
+    '/topics/:abbr/:category?/:option?': ['axojhf'],
+    '/schedules/:serie/:category?': ['axojhf'],
 };

package/lib/v2/lovelive-anime/radar.js

@@ -17,6 +17,15 @@ module.exports = {
                     return `/lovelive-anime/topics/${params.abbr}/${null !== cat && cat.length === 2 ? cat[1] : ''}`;
                 },
             },
+            {
+                title: 'Schedule',
+                docs: 'https://docs.rsshub.app/anime.html#lovelive-anime-love-live-guan-wang-schedule',
+                source: ['/schedule/'],
+                target: (params, url) => {
+                    const cat = url.match(/\?series=(.*)&category=(.*)/);
+                    return `/lovelive-anime/schedules/${null !== cat && cat.length >= 2 ? cat[1] : ''}/${null !== cat && cat.length === 3 ? cat[2] : ''}`;
+                },
+            },
         ],
     },
 };

package/lib/v2/lovelive-anime/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
     router.get('/news/:option?', require('./news'));
     router.get('/topics/:abbr/:category?/:option?', require('./topics'));
+    router.get('/schedules/:serie?/:category?', require('./schedules'));
 };

package/lib/v2/lovelive-anime/schedules.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const path = require('path');
+const { art } = require('@/utils/render');
+const renderDescription = (desc) => art(path.join(__dirname, 'templates/scheduleDesc.art'), desc);
+const dayjs = require('dayjs');
+const utc = require('dayjs/plugin/utc');
+const timezone = require('dayjs/plugin/timezone');
+
+module.exports = async (ctx) => {
+    dayjs.extend(utc);
+    dayjs.extend(timezone);
+    const serie = ctx.params.serie;
+    const category = ctx.params.category;
+    const rootUrl = `https://www.lovelive-anime.jp/common/api/calendar_list.php`;
+    const nowTime = dayjs();
+    const dataPara = {
+        year: nowTime.year(),
+        month: nowTime.format('MM'),
+    };
+    if (serie && 'all' !== serie) {
+        dataPara.series = [serie];
+    }
+    if (category && 'all' !== category) {
+        dataPara.category = [category];
+    }
+    const response = await got(`${rootUrl}?site=jp&ip=lovelive&data=${JSON.stringify(dataPara)}`);
+
+    const items = response.data.data.schedule_list.map((item) => {
+        const link = item.url.select_url;
+        const title = item.title;
+        const category = item.categories.name;
+        const eventStartDate = dayjs.unix(item.event_startdate).tz('Asia/Tokyo').format('YYYY-MM-DD HH:mm:ss JST');
+        const eventEndDate = dayjs.unix(item.event_enddate).tz('Asia/Tokyo').format('YYYY-MM-DD HH:mm:ss JST');
+
+        return {
+            link,
+            pubDate: item.start_endflg ? dayjs.unix(item.event_enddate).tz('Asia/Tokyo') : dayjs.unix(item.event_startdate).tz('Asia/Tokyo'),
+            title,
+            category,
+            description: renderDescription({
+                title,
+                desc: item.event_dspdate,
+                startTime: eventStartDate,
+                endTime: eventEndDate,
+            }),
+        };
+    });
+
+    ctx.state.data = {
+        title: `${category} - ${serie} - Love Live Official Website Schedule`,
+        link: 'https://www.lovelive-anime.jp/schedule/',
+        item: items,
+        description:
+            'The schedule date and other information shall be subject to the announcement on the official website. The RSS route has not been strictly tested, and the captured information cannot be guaranteed to be correct.',
+    };
+};

package/lib/v2/lovelive-anime/templates/scheduleDesc.art

@@ -0,0 +1,3 @@
+<h1>{{title}}</h1><br>
+<b>{{startTime}}</b>&nbsp;&nbsp;&nbsp;~&nbsp;&nbsp;&nbsp;<b>{{endTime}}</b><br><br>
+{{desc}}

package/lib/v2/luogu/daily.js

@@ -21,7 +21,7 @@ module.exports = async (ctx) => {
             link,
             author: firstPost.find('p').eq(1).text(),
             guid: `${link}#${issueHeading}`,
-            pubDate: parseDate(issueHeading.match(/(\d{4} 年 \d{2} 月 \d{2} 日)/)[1], 'YYYY 年 MM 月 DD 日'),
+            pubDate: parseDate(issueHeading.match(/(\d{4} 年 \d{1,2} 月 \d{1,2} 日)/)[1], 'YYYY 年 M 月 D 日'),
         },
     ];
 

package/lib/v2/luogu/maintainer.js

@@ -2,4 +2,5 @@ module.exports = {
     '/contest': ['prnake'],
     '/daily/:id?': ['LogicJake ', 'prnake ', 'nczitzk'],
     '/user/feed/:uid': ['solstice23'],
+    '/user/blog/:username': ['ftiasch'],
 };

package/lib/v2/luogu/radar.js

@@ -20,6 +20,12 @@ module.exports = {
                 source: ['/user/:uid'],
                 target: '/luogu/user/feed/:uid',
             },
+            {
+                title: '用户博客',
+                docs: 'https://docs.rsshub.app/programming.html#luo-gu',
+                source: ['/blog/:name'],
+                target: '/luogu/user/blog/:name',
+            },
         ],
     },
 };

package/lib/v2/luogu/router.js

@@ -2,4 +2,5 @@ module.exports = (router) => {
     router.get('/contest', require('./contest'));
     router.get('/daily/:id?', require('./daily'));
     router.get('/user/feed/:uid', require('./userFeed'));
+    router.get('/user/blog/:name', require('./userBlog'));
 };

package/lib/v2/luogu/userBlog.js

@@ -0,0 +1,48 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const name = ctx.params.name;
+
+    const blogBaseUrl = `https://www.luogu.com.cn/blog/${name}/`;
+
+    // Fetch the uid & title
+    const { uid: blogUid, title: blogTitle } = await ctx.cache.tryGet(blogBaseUrl, async () => {
+        const rsp = await got(blogBaseUrl);
+        const $ = cheerio.load(rsp.data);
+        const uid = $("meta[name='blog-uid']").attr('content');
+        const name = $("meta[name='blog-name']").attr('content');
+        return {
+            uid,
+            title: `${name} - 洛谷博客`,
+        };
+    });
+
+    const posts = (await got(`https://www.luogu.com.cn/api/blog/lists?uid=${blogUid}`).json()).data.result.map((r) => ({
+        title: r.title,
+        link: `${blogBaseUrl}${r.identifier}`,
+        pubDate: new Date(r.postTime * 1000),
+    }));
+
+    // Get the full text
+    const item = await Promise.all(
+        posts.map((post) =>
+            ctx.cache.tryGet(post.link, async () => {
+                const rsp = await got(post.link);
+                const $ = cheerio.load(rsp.data);
+                return {
+                    title: post.title,
+                    link: post.link,
+                    pubDate: post.pubDate,
+                    description: $('#article-content').html(),
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: blogTitle,
+        link: blogBaseUrl,
+        item,
+    };
+};

package/lib/v2/mangadex/index.js

@@ -0,0 +1,73 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { RequestInProgressError } = require('@/errors');
+
+module.exports = async (ctx) => {
+    const { id, lang } = ctx.params;
+    let data = (await got.get(`https://api.mangadex.org/manga/${id}`)).data;
+    if (data.result === 'error') {
+        throw new RequestInProgressError(data.errors[0].detail);
+    }
+    data = data.data;
+    let title;
+    if (lang) {
+        title = data.attributes.title[lang];
+        if (!title) {
+            title = data.attributes.altTitles.find((altTitle) => altTitle[lang])?.[lang];
+        }
+    }
+    if (!title) {
+        title = data.attributes.title[data.attributes.originalLanguage];
+        if (!title) {
+            title = data.attributes.altTitles.find((altTitle) => altTitle[data.attributes.originalLanguage])?.[data.attributes.originalLanguage];
+            if (!title) {
+                title = Object.values(data.attributes.title)[0];
+            }
+        }
+    }
+    let description;
+    if (lang) {
+        description = data.attributes.description[lang];
+    }
+    if (!description) {
+        description = data.attributes.description[data.attributes.originalLanguage];
+        if (!description) {
+            description = Object.values(data.attributes.description)[0];
+        }
+    }
+
+    let url = `https://api.mangadex.org/manga/${id}/feed?order[publishAt]=desc`;
+    if (lang) {
+        url += `&translatedLanguage[]=${lang}`;
+    }
+    data = (await got.get(url)).data;
+    if (data.result === 'error') {
+        throw new RequestInProgressError(data.errors[0].detail);
+    }
+    ctx.state.data = {
+        title: `${title} - MangaDex`,
+        link: `https://mangadex.org/title/${id}`,
+        description,
+        allowEmpty: true,
+        item: data.data.map((chapter) => {
+            const title = [];
+            if (chapter.attributes.volume) {
+                title.push(`Vol. ${chapter.attributes.volume}`);
+            }
+            if (chapter.attributes.chapter) {
+                title.push(`Ch. ${chapter.attributes.chapter}`);
+            }
+            if (chapter.attributes.title) {
+                title.push(chapter.attributes.title);
+            }
+            return {
+                title: title.join(' '),
+                link: `https://mangadex.org/chapter/${chapter.id}`,
+                pubDate: parseDate(chapter.attributes.publishAt),
+            };
+        }),
+    };
+    if (lang) {
+        ctx.state.data.language = lang;
+    }
+};

package/lib/v2/mangadex/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:id/:lang?': ['vzz64'],
+};

package/lib/v2/mangadex/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'mangadex.org': {
+        _name: 'MangaDex',
+        '.': [
+            {
+                title: '漫画更新',
+                docs: 'https://docs.rsshub.app/anime.html#mangadex',
+                source: ['/title/:id/*', '/title/:id'],
+                target: '/mangadex/:id',
+            },
+        ],
+    },
+};

package/lib/v2/mangadex/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:id/:lang?', require('./index'));
+};