rsshub 1.0.0-master.f961961 → 1.0.0-master.f992838

package/lib/v2/linkedin/utils.js

@@ -0,0 +1,121 @@
+const cheerio = require('cheerio');
+const { Job } = require('./models');
+
+/**
+ * Constants
+ */
+const KEYWORDS_QUERY_KEY = 'keywords';
+
+const JOB_TYPES_QUERY_KEY = 'f_JT';
+
+const JOB_TYPES = {
+    F: 'full-time',
+    P: 'part-time',
+    C: 'contract',
+};
+
+const EXP_LEVELS_QUERY_KEY = 'f_E';
+
+const EXP_LEVELS = {
+    1: 'internship',
+    2: 'entry',
+    3: 'associate',
+    4: 'mid-senior',
+    5: 'director',
+};
+
+/**
+ * Params parsing
+ */
+
+/**
+ * Parse params in route into valid query params
+ * e.g. /jobs/:keywords/:job_types/:exp_levels
+ *   if job_types is 'C-T'
+ *   the output search param is 'C,T', which can be
+ *   used as search param in query
+ * @param {String} params params in route (separated by '-')
+ * @param {Object} map valid value map for this param
+ * @returns search params separated by ',' that can be used
+ *   as search param in url
+ */
+function parseParamsToSearchParams(params, map) {
+    const validParamValues = params.split('-').filter((v) => v in map);
+    return validParamValues.join(',');
+}
+
+/**
+ * Parse params in route into human readable strings
+ * e.g. /jobs/:keywords/:job_types/:exp_levels
+ *   if job_types is 'C-T'
+ *   the output is 'contract,full-time'
+ * @param {String} params params in route (separated by '-')
+ * @param {Object} map valid value map for this param
+ * @returns param value strings separated by ','
+ */
+function parseParamsToString(params, map) {
+    const validParamValues = params
+        .split('-')
+        .filter((v) => v in map)
+        .map((v) => map[v]);
+    return validParamValues.join(',');
+}
+
+/**
+ * HTML page parsing
+ */
+
+/**
+ * Parse job search page
+ * Example page: https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?keywords=Software%20Engineer&location=United%20States&locationId=&geoId=103644278&sortBy=R&f_TPR=&position=1&pageNum=0
+ *
+ * @param {String} data HTML string of job search page
+ * @returns {Job[]} Array of jobs with data filled
+ */
+function parseJobSearch(data) {
+    const $ = cheerio.load(data);
+
+    // Parse data
+    const jobs = $('li')
+        .map((i, elem) => {
+            const elemHtml = $(elem);
+            const link = elemHtml.find('a.base-card__full-link').attr('href').split('?')[0];
+            const title = elemHtml.find('h3.base-search-card__title').text().trim();
+            const company = elemHtml.find('h4.base-search-card__subtitle').text().trim();
+            const location = elemHtml.find('span.job-search-card__location').text().trim();
+            const pubDate = elemHtml.find('time').attr('datetime');
+
+            return new Job(title, link, company, location, pubDate);
+        })
+        .toArray();
+    return jobs;
+}
+
+/**
+ * Parse job detail page
+ * Example page: https://www.linkedin.com/jobs/view/software-engineer-backend-junior-at-genies-3429649821?trk=public_jobs_topcard-title
+ *
+ * @param {String} data HTML string of job detail page
+ * @returns {Job} Job details
+ */
+function parseJobDetail(data) {
+    const job = new Job();
+    const $ = cheerio.load(data);
+
+    job.recruiter = $('a.message-the-recruiter__cta').attr(`href`);
+    job.description = $('div.description__text description__text--rich').text();
+
+    return job;
+}
+
+module.exports = {
+    parseParamsToSearchParams,
+    parseParamsToString,
+    parseJobDetail,
+    parseJobSearch,
+    JOB_TYPES,
+    JOB_TYPES_QUERY_KEY,
+    EXP_LEVELS,
+    EXP_LEVELS_QUERY_KEY,
+    KEYWORDS_QUERY_KEY,
+};

package/lib/v2/lofter/user.js

@@ -1,9 +1,13 @@
 const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const name = ctx.params.name ?? 'i';
     const limit = ctx.query.limit ? parseInt(ctx.query.limit) : '50';
+    if (!isValidHost(name)) {
+        throw Error('Invalid name');
+    }
 
     const rootUrl = `${name}.lofter.com`;
 
@@ -33,10 +37,10 @@ module.exports = async (ctx) => {
         description:
             JSON.parse(item.post.photoLinks || `[]`)
                 .map((photo) => {
-                    if (photo.raw.match(/\/\/nos\.netease\.com\//)) {
+                    if (photo.raw?.match(/\/\/nos\.netease\.com\//)) {
                         photo.raw = `https://${photo.raw.match(/(imglf\d)/)[0]}.lf127.net${photo.raw.match(/\/\/nos\.netease\.com\/imglf\d(.*)/)[1]}`;
                     }
-                    return `<img src="${photo.raw}">`;
+                    return `<img src="${photo.raw || photo.orign}">`;
                 })
                 .join('') +
             JSON.parse(item.post.embed ? `[${item.post.embed}]` : `[]`)

package/lib/v2/luogu/maintainer.js

@@ -2,4 +2,5 @@ module.exports = {
     '/contest': ['prnake'],
     '/daily/:id?': ['LogicJake ', 'prnake ', 'nczitzk'],
     '/user/feed/:uid': ['solstice23'],
+    '/user/blog/:username': ['ftiasch'],
 };

package/lib/v2/luogu/radar.js

@@ -20,6 +20,12 @@ module.exports = {
                 source: ['/user/:uid'],
                 target: '/luogu/user/feed/:uid',
             },
+            {
+                title: '用户博客',
+                docs: 'https://docs.rsshub.app/programming.html#luo-gu',
+                source: ['/blog/:name'],
+                target: '/luogu/user/blog/:name',
+            },
         ],
     },
 };

package/lib/v2/luogu/router.js

@@ -2,4 +2,5 @@ module.exports = (router) => {
     router.get('/contest', require('./contest'));
     router.get('/daily/:id?', require('./daily'));
     router.get('/user/feed/:uid', require('./userFeed'));
+    router.get('/user/blog/:name', require('./userBlog'));
 };

package/lib/v2/luogu/userBlog.js

@@ -0,0 +1,48 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const name = ctx.params.name;
+
+    const blogBaseUrl = `https://www.luogu.com.cn/blog/${name}/`;
+
+    // Fetch the uid & title
+    const { uid: blogUid, title: blogTitle } = await ctx.cache.tryGet(blogBaseUrl, async () => {
+        const rsp = await got(blogBaseUrl);
+        const $ = cheerio.load(rsp.data);
+        const uid = $("meta[name='blog-uid']").attr('content');
+        const name = $("meta[name='blog-name']").attr('content');
+        return {
+            uid,
+            title: `${name} - 洛谷博客`,
+        };
+    });
+
+    const posts = (await got(`https://www.luogu.com.cn/api/blog/lists?uid=${blogUid}`).json()).data.result.map((r) => ({
+        title: r.title,
+        link: `${blogBaseUrl}${r.identifier}`,
+        pubDate: new Date(r.postTime * 1000),
+    }));
+
+    // Get the full text
+    const item = await Promise.all(
+        posts.map((post) =>
+            ctx.cache.tryGet(post.link, async () => {
+                const rsp = await got(post.link);
+                const $ = cheerio.load(rsp.data);
+                return {
+                    title: post.title,
+                    link: post.link,
+                    pubDate: post.pubDate,
+                    description: $('#article-content').html(),
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: blogTitle,
+        link: blogBaseUrl,
+        item,
+    };
+};

package/lib/v2/mangadex/index.js

@@ -0,0 +1,73 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { RequestInProgressError } = require('@/errors');
+
+module.exports = async (ctx) => {
+    const { id, lang } = ctx.params;
+    let data = (await got.get(`https://api.mangadex.org/manga/${id}`)).data;
+    if (data.result === 'error') {
+        throw new RequestInProgressError(data.errors[0].detail);
+    }
+    data = data.data;
+    let title;
+    if (lang) {
+        title = data.attributes.title[lang];
+        if (!title) {
+            title = data.attributes.altTitles.find((altTitle) => altTitle[lang])?.[lang];
+        }
+    }
+    if (!title) {
+        title = data.attributes.title[data.attributes.originalLanguage];
+        if (!title) {
+            title = data.attributes.altTitles.find((altTitle) => altTitle[data.attributes.originalLanguage])?.[data.attributes.originalLanguage];
+            if (!title) {
+                title = Object.values(data.attributes.title)[0];
+            }
+        }
+    }
+    let description;
+    if (lang) {
+        description = data.attributes.description[lang];
+    }
+    if (!description) {
+        description = data.attributes.description[data.attributes.originalLanguage];
+        if (!description) {
+            description = Object.values(data.attributes.description)[0];
+        }
+    }
+
+    let url = `https://api.mangadex.org/manga/${id}/feed?order[publishAt]=desc`;
+    if (lang) {
+        url += `&translatedLanguage[]=${lang}`;
+    }
+    data = (await got.get(url)).data;
+    if (data.result === 'error') {
+        throw new RequestInProgressError(data.errors[0].detail);
+    }
+    ctx.state.data = {
+        title: `${title} - MangaDex`,
+        link: `https://mangadex.org/title/${id}`,
+        description,
+        allowEmpty: true,
+        item: data.data.map((chapter) => {
+            const title = [];
+            if (chapter.attributes.volume) {
+                title.push(`Vol. ${chapter.attributes.volume}`);
+            }
+            if (chapter.attributes.chapter) {
+                title.push(`Ch. ${chapter.attributes.chapter}`);
+            }
+            if (chapter.attributes.title) {
+                title.push(chapter.attributes.title);
+            }
+            return {
+                title: title.join(' '),
+                link: `https://mangadex.org/chapter/${chapter.id}`,
+                pubDate: parseDate(chapter.attributes.publishAt),
+            };
+        }),
+    };
+    if (lang) {
+        ctx.state.data.language = lang;
+    }
+};

package/lib/v2/mangadex/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:id/:lang?': ['vzz64'],
+};

package/lib/v2/mangadex/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'mangadex.org': {
+        _name: 'MangaDex',
+        '.': [
+            {
+                title: '漫画更新',
+                docs: 'https://docs.rsshub.app/anime.html#mangadex',
+                source: ['/title/:id/*', '/title/:id'],
+                target: '/mangadex/:id',
+            },
+        ],
+    },
+};

package/lib/v2/mangadex/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:id/:lang?', require('./index'));
+};

package/lib/v2/minecraft/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/version': ['TheresaQWQ'],
+};

package/lib/v2/minecraft/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'minecraft.net': {
+        _name: '我的世界',
+        '.': [
+            {
+                title: 'Java 版游戏更新',
+                docs: 'https://docs.rsshub.app/game.html#minecraft',
+                source: ['/'],
+                target: '/minecraft/version',
+            },
+        ],
+    },
+};

package/lib/v2/minecraft/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/version', require('./version'));
+};

package/lib/{routes → v2}/minecraft/version.js

@@ -28,7 +28,7 @@ module.exports = async (ctx) => {
             description: `${item.id} ${typeMap[item.type] || ''}更新`,
             pubDate: new Date(item.releaseTime).toUTCString(),
             link: `https://www.minecraft.net`,
-            guid: item.url,
+            guid: item.id + item.type,
         })),
     };
 };

package/lib/v2/mingpao/index.js

@@ -5,6 +5,17 @@ const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
 
+const renderFanBox = (media) =>
+    art(path.join(__dirname, 'templates/fancybox.art'), {
+        media,
+    });
+
+const renderDesc = (media, desc) =>
+    art(path.join(__dirname, 'templates/description.art'), {
+        media: renderFanBox(media),
+        desc,
+    });
+
 module.exports = async (ctx) => {
     const type = ctx.params.type ?? 'ins';
     const category = ctx.params.category ?? (type === 'ins' ? 'all' : 's00001');
@@ -12,23 +23,10 @@ module.exports = async (ctx) => {
 
     const feed = await parser.parseURL(link);
 
-    const renderFanBox = (pics) =>
-        art(path.join(__dirname, 'templates/fancybox.art'), {
-            pics,
-        });
-
-    const renderDesc = (pics, desc) =>
-        art(path.join(__dirname, 'templates/description.art'), {
-            pics: renderFanBox(pics),
-            desc,
-        });
-
     const items = await Promise.all(
         feed.items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const response = await got({
-                    method: 'get',
-                    url: item.link,
+                const response = await got(item.link, {
                     headers: {
                         Referer: 'https://news.mingpao.com/',
                     },
@@ -48,15 +46,21 @@ module.exports = async (ctx) => {
                 item.category = item.categories;
 
                 // fix fancybox image
-                const fancybox = fancyboxImg
-                    .map((_, e) => {
-                        e = $(e);
-                        return {
-                            href: e.attr('href'),
-                            title: e.attr('title'),
-                        };
-                    })
-                    .get();
+                const fancybox = fancyboxImg.toArray().map((e) => {
+                    e = $(e);
+                    const href = new URL(e.attr('href'));
+                    let video;
+                    if (href.hostname === 'videop.mingpao.com') {
+                        video = new URL(href.searchParams.get('file'));
+                        video.hostname = 'cfrvideo.mingpao.com'; // use cloudflare cdn
+                        video = video.href;
+                    }
+                    return {
+                        href: href.href,
+                        title: e.attr('title'),
+                        video,
+                    };
+                });
 
                 // remove unwanted key value
                 delete item.categories;
@@ -65,7 +69,7 @@ module.exports = async (ctx) => {
                 delete item.creator;
                 delete item.isoDate;
 
-                item.description = renderDesc(fancybox, $('.txt4').html() ?? $('div.article_content.line_1_5em').html());
+                item.description = renderDesc(fancybox, $('.txt4').html() ?? $('.article_content.line_1_5em').html());
                 item.pubDate = parseDate(item.pubDate);
                 item.guid = item.link.indexOf('?') > 0 ? item.link : item.link.substring(0, item.link.lastIndexOf('/'));
 
@@ -82,13 +86,4 @@ module.exports = async (ctx) => {
         image: feed.image.url,
         language: feed.language,
     };
-
-    ctx.state.json = {
-        title: feed.title,
-        link: feed.link,
-        description: feed.description,
-        item: items,
-        image: feed.image.url,
-        language: feed.language,
-    };
 };

package/lib/v2/mingpao/templates/description.art

@@ -1,2 +1,2 @@
-{{@ pics }}
+{{@ media }}
 {{@ desc }}

package/lib/v2/mingpao/templates/fancybox.art

@@ -1,3 +1,7 @@
-{{ each pics }}
-    <figure><img src="{{ $value.href }}" alt="{{ $value.title }}"><figcaption>{{ $value.title }}</figcaption></figure>
+{{ each media }}
+    {{ if $value.video }}
+        <video controls preload="none" poster="{{ $value.video.replace('.mp4', '.jpg') }}"><source src="{{ $value.video }}" type="video/mp4"></video>
+    {{ else }}
+        <figure><img src="{{ $value.href }}" alt="{{ $value.title }}"><figcaption>{{ $value.title }}</figcaption></figure>
+    {{ /if }}
 {{ /each }}

package/lib/v2/mirror/index.js

@@ -4,10 +4,13 @@ const md = require('markdown-it')({
     html: true,
     linkify: true,
 });
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
-
+    if (!id.endsWith('.eth') && !isValidHost(id)) {
+        throw Error('Invalid id');
+    }
     const rootUrl = 'https://mirror.xyz';
     const currentUrl = id.endsWith('.eth') ? `${rootUrl}/${id}` : `https://${id}.mirror.xyz`;
 

package/lib/v2/mixcloud/index.js

@@ -0,0 +1,130 @@
+const got = require('@/utils/got');
+const CryptoJS = require('crypto-js');
+const { parseDate } = require('@/utils/parse-date');
+const { queries } = require('./queries');
+
+module.exports = async (ctx) => {
+    const host = 'https://www.mixcloud.com';
+    const imageBaseURL = 'https://thumbnailer.mixcloud.com/unsafe/480x480/';
+    const headers = {
+        Referer: host,
+        'Content-Type': 'application/json',
+        'X-Requested-With': 'XMLHttpRequest',
+    };
+
+    const type = ctx.params.type ?? 'uploads';
+    if (!['stream', 'uploads', 'favorites', 'listens'].includes(type)) {
+        throw Error(`Invalid type: ${type}`);
+    }
+    const username = ctx.params.username;
+
+    const config = {
+        stream: { name: 'Stream', node: 'stream' },
+        uploads: { name: 'Shows', node: 'uploads' },
+        favorites: { name: 'Favorites', node: 'favorites' },
+        listens: { name: 'History', node: 'listeningHistory' },
+    };
+    const payloads = {
+        stream: {
+            query: queries.stream.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+            },
+        },
+        uploads: {
+            query: queries.uploads.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+                orderBy: 'LATEST',
+                audioTypes: ['SHOW'],
+            },
+        },
+        favorites: {
+            query: queries.favorites.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+            },
+        },
+        listens: {
+            query: queries.listens.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+            },
+        },
+        profile: {
+            query: queries.profile.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+            },
+        },
+    };
+
+    const profile = (
+        await got({
+            method: 'post',
+            url: `${host}/graphql`,
+            json: payloads.profile,
+            headers,
+        })
+    ).data.data;
+
+    const biog = profile.user.biog;
+    const image = `${imageBaseURL}${profile.user.picture.urlRoot}`;
+
+    const data = (
+        await got({
+            method: 'post',
+            url: `${host}/graphql`,
+            json: payloads[type],
+            headers,
+        })
+    ).data.data;
+
+    // https://github.com/ytdl-org/youtube-dl/blob/f1487d4fca40fd37d735753e24a7bae53a1b1513/youtube_dl/extractor/mixcloud.py#L72-L79
+    const decryptionKey = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD';
+    const decryptXorCipher = (key, cipherText) => {
+        const cipher = CryptoJS.enc.Base64.parse(cipherText);
+        const decrypted = cipher.toString(CryptoJS.enc.Utf8);
+        let result = '';
+        for (let i = 0; i < decrypted.length; i++) {
+            result += String.fromCharCode(decrypted.charCodeAt(i) ^ key.charCodeAt(i % key.length));
+        }
+        return result;
+    };
+
+    const items = data.user[config[type].node].edges.map((edge) => {
+        const item = type === 'listens' ? edge.node.cloudcast : edge.node;
+        return {
+            title: item.name,
+            author: item.owner.displayName,
+            description: item.description.replace(/\n/g, '<br>'),
+            pubDate: parseDate(item.publishDate),
+            guid: Buffer.from(item.id, 'base64').toString('utf8'),
+            link: `${host}/${username}/${item.slug}`,
+            itunes_item_image: `${imageBaseURL}${item.picture.urlRoot}`,
+            itunes_duration: item.audioLength,
+            enclosure_url: decryptXorCipher(decryptionKey, item.streamInfo.url),
+            enclosure_type: 'audio/x-m4a',
+            upvotes: item.favorites.totalCount,
+        };
+    });
+
+    ctx.state.data = {
+        title: `Mixcloud - ${data.user.displayName}'s ${config[type].name}`,
+        description: biog.replace(/\n/g, '<br>'),
+        itunes_author: data.user.displayName,
+        image,
+        link: `${host}/${username}/${type}/`,
+        item: items,
+    };
+};

package/lib/v2/mixcloud/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:username/:type?': ['Misaka13514'],
+};