rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/lib/v2/linkedin/models.js

@@ -0,0 +1,15 @@
+class Job {
+    constructor(title, link, company, location, pubDate, recruiter, description) {
+        this.title = title;
+        this.link = link;
+        this.company = company;
+        this.location = location;
+        this.pubDate = pubDate;
+        this.recruiter = recruiter;
+        this.description = description;
+    }
+}
+
+module.exports = {
+    Job,
+};

package/lib/v2/linkedin/radar.js

@@ -0,0 +1,36 @@
+module.exports = {
+    'linkedin.com': {
+        _name: 'LinkedIn',
+        '.': [
+            {
+                title: 'Job Listing',
+                docs: 'https://docs.rsshub.app/en/other.html#linkedin-jobs',
+                source: '/jobs/search/',
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    const parseRoute = (searchParam) => {
+                        if (typeof searchParam !== 'string') {
+                            return 'all';
+                        }
+                        return searchParam.split(',').join('-');
+                    };
+                    return `/linkedin/jobs/${parseRoute(searchParams.get('f_JT'))}/${parseRoute(searchParams.get('f_E'))}/${searchParams.get('keywords') || ''}`;
+                },
+            },
+        ],
+    },
+    'linkedin.cn': {
+        _name: 'LinkedIn 领英中国',
+        '.': [
+            {
+                title: 'Jobs',
+                docs: 'https://docs.rsshub.app/other.html#linkedin-ling-ying-zhong-guo',
+                source: '/incareer/jobs/search',
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    return `/linkedin/cn/jobs/${searchParams.get('keywords') || ''}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/linkedin/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/cn/jobs/:keywords?', require('./cn/index.js'));
+    router.get('/jobs/:job_types/:exp_levels/:keywords?', require('./jobs.js'));
+};

package/lib/v2/linkedin/templates/cn/posting.art

@@ -0,0 +1,25 @@
+<h2>{{ title }}</h2>
+<p>
+    {{ if (applyMethod.instantOffsiteApply)}}
+    <a href='{{applyMethod.instantOffsiteApply.companyApplyUrl}}'>点击申请</a>
+    {{ /if }}
+</p>
+<p>
+    {{ if (applyMethod.basicOffsiteApply)}}
+    <a href='{{applyMethod.basicOffsiteApply.companyApplyUrl}}'>点击申请</a>
+    {{ /if }}
+</p>
+<p>已有{{numApplies}}人申请此职位， {{numViews}}人查看此职位</p>
+{{ if(compensationDescription) }}
+<p>薪资：{{ compensationDescription || 'N/A' }} </p>
+{{ /if }}
+<p>工作地点： {{ geo.defaultLocalizedName }} </p>
+
+{{ if (company) }}
+<h2>公司介绍</h2>
+<p>{{company.name}}</p>
+<p>员工人数：{{company.employeeCount}}</p>
+<p>{{company.localizedDescription}}</p>
+{{ /if }}
+<h2>职位介绍</h2>
+<article>{{@ desc}}</article>

package/lib/v2/linkedin/utils.js

@@ -0,0 +1,121 @@
+const cheerio = require('cheerio');
+const { Job } = require('./models');
+
+/**
+ * Constants
+ */
+const KEYWORDS_QUERY_KEY = 'keywords';
+
+const JOB_TYPES_QUERY_KEY = 'f_JT';
+
+const JOB_TYPES = {
+    F: 'full-time',
+    P: 'part-time',
+    C: 'contract',
+};
+
+const EXP_LEVELS_QUERY_KEY = 'f_E';
+
+const EXP_LEVELS = {
+    1: 'internship',
+    2: 'entry',
+    3: 'associate',
+    4: 'mid-senior',
+    5: 'director',
+};
+
+/**
+ * Params parsing
+ */
+
+/**
+ * Parse params in route into valid query params
+ * e.g. /jobs/:keywords/:job_types/:exp_levels
+ *   if job_types is 'C-T'
+ *   the output search param is 'C,T', which can be
+ *   used as search param in query
+ * @param {String} params params in route (separated by '-')
+ * @param {Object} map valid value map for this param
+ * @returns search params separated by ',' that can be used
+ *   as search param in url
+ */
+function parseParamsToSearchParams(params, map) {
+    const validParamValues = params.split('-').filter((v) => v in map);
+    return validParamValues.join(',');
+}
+
+/**
+ * Parse params in route into human readable strings
+ * e.g. /jobs/:keywords/:job_types/:exp_levels
+ *   if job_types is 'C-T'
+ *   the output is 'contract,full-time'
+ * @param {String} params params in route (separated by '-')
+ * @param {Object} map valid value map for this param
+ * @returns param value strings separated by ','
+ */
+function parseParamsToString(params, map) {
+    const validParamValues = params
+        .split('-')
+        .filter((v) => v in map)
+        .map((v) => map[v]);
+    return validParamValues.join(',');
+}
+
+/**
+ * HTML page parsing
+ */
+
+/**
+ * Parse job search page
+ * Example page: https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?keywords=Software%20Engineer&location=United%20States&locationId=&geoId=103644278&sortBy=R&f_TPR=&position=1&pageNum=0
+ *
+ * @param {String} data HTML string of job search page
+ * @returns {Job[]} Array of jobs with data filled
+ */
+function parseJobSearch(data) {
+    const $ = cheerio.load(data);
+
+    // Parse data
+    const jobs = $('li')
+        .map((i, elem) => {
+            const elemHtml = $(elem);
+            const link = elemHtml.find('a.base-card__full-link').attr('href').split('?')[0];
+            const title = elemHtml.find('h3.base-search-card__title').text().trim();
+            const company = elemHtml.find('h4.base-search-card__subtitle').text().trim();
+            const location = elemHtml.find('span.job-search-card__location').text().trim();
+            const pubDate = elemHtml.find('time').attr('datetime');
+
+            return new Job(title, link, company, location, pubDate);
+        })
+        .toArray();
+    return jobs;
+}
+
+/**
+ * Parse job detail page
+ * Example page: https://www.linkedin.com/jobs/view/software-engineer-backend-junior-at-genies-3429649821?trk=public_jobs_topcard-title
+ *
+ * @param {String} data HTML string of job detail page
+ * @returns {Job} Job details
+ */
+function parseJobDetail(data) {
+    const job = new Job();
+    const $ = cheerio.load(data);
+
+    job.recruiter = $('a.message-the-recruiter__cta').attr(`href`);
+    job.description = $('div.description__text description__text--rich').text();
+
+    return job;
+}
+
+module.exports = {
+    parseParamsToSearchParams,
+    parseParamsToString,
+    parseJobDetail,
+    parseJobSearch,
+    JOB_TYPES,
+    JOB_TYPES_QUERY_KEY,
+    EXP_LEVELS,
+    EXP_LEVELS_QUERY_KEY,
+    KEYWORDS_QUERY_KEY,
+};

package/lib/v2/lofter/user.js

@@ -1,9 +1,13 @@
 const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const name = ctx.params.name ?? 'i';
     const limit = ctx.query.limit ? parseInt(ctx.query.limit) : '50';
+    if (!isValidHost(name)) {
+        throw Error('Invalid name');
+    }
 
     const rootUrl = `${name}.lofter.com`;
 
@@ -33,10 +37,10 @@ module.exports = async (ctx) => {
         description:
             JSON.parse(item.post.photoLinks || `[]`)
                 .map((photo) => {
-                    if (photo.raw.match(/\/\/nos\.netease\.com\//)) {
+                    if (photo.raw?.match(/\/\/nos\.netease\.com\//)) {
                         photo.raw = `https://${photo.raw.match(/(imglf\d)/)[0]}.lf127.net${photo.raw.match(/\/\/nos\.netease\.com\/imglf\d(.*)/)[1]}`;
                     }
-                    return `<img src="${photo.raw}">`;
+                    return `<img src="${photo.raw || photo.orign}">`;
                 })
                 .join('') +
             JSON.parse(item.post.embed ? `[${item.post.embed}]` : `[]`)

package/lib/v2/mclaren/index.js

@@ -0,0 +1,95 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+module.exports = async (ctx) => {
+  let baseUrl, apiUrl;
+  let title, description;
+  let lang = ctx.params.lang;
+  switch (lang) {
+	case 'en':
+	  baseUrl = 'https://www.mclaren.com';
+	  title = 'McLaren Racing – Official Website';
+	  description = 'Latest news and insight from McLaren Racing. Team and driver updates, videos and McLaren Formula 1 LIVE commentary.';
+	  lang = 'en-UK';
+	  break;
+	case 'zh':
+	  baseUrl = 'https://cn.mclaren.com';
+	  title = '迈凯伦赛车 — 官方网站';
+	  description = '来自迈凯伦一级方程式车队的最新消息。 查看车队和车手近况、视频以及比赛中的迈凯伦实况评论和实时数据。';
+	  lang = 'zh-CN';
+	  break;
+	case 'es':
+	  baseUrl = 'https://es.mclaren.com';
+	  title = 'McLaren Racing - Sitio web oficial';
+	  description = 'Consulta las últimas noticias del equipo de Fórmula 1 McLaren. Incluye información sobre el equipo, las últimas actualizaciones de los pilotos y McLaren LIVE, una aplicación con la que podrás seguir los comentarios en directo.';
+	  lang = 'es-ES';
+	  break;
+	default:
+	  throw Error(`Language '${lang}' is not supported.`);
+  };
+
+  const { category = 'all' } = ctx.params;
+  const categoryArr = ['all', 'article', 'report', 'gallery', 'video', 'blog', 'photo_essay'];
+  if (!categoryArr.includes(category)) {
+	throw Error(`Category '${category}' is not supported.`);
+  };
+
+  const link = baseUrl + '/racing/articles/';
+  apiUrl = baseUrl + '/racing/api/grid-search/?count=30&offset=0&type=';
+  // count: number of requested entries; default to 16, up to 30.
+  // offset: default to 0.
+  if (category !== 'all') {
+	apiUrl += category;
+  }
+
+  const response = await got(apiUrl);
+
+  const html = response.data.html;
+  const $ = cheerio.load(html);
+  const list = $('li')
+	.toArray()
+	.map((item) => {
+	  item = $(item);
+	  const thumbnailUrl = item.find('div.start-grid-content__img').attr('style').slice(22, -2);
+	  return {
+		link: baseUrl + item.find('a.start-grid-content__link').attr('href'),
+		thumbnailUrl, // Pass the thumbnail image
+	  };
+	});
+
+  const items = await Promise.all(
+	list.map((item) =>
+	  ctx.cache.tryGet(item.link, async () => {
+		const itemRep = await got(item.link);
+		const $ = cheerio.load(itemRep.data);
+		$('.livestream--wrapper').last().remove();
+		item.title = $('meta[property="og:title"]').attr('content').slice(17);
+		item.pubDate = Date.parse($('time[datetime]').first().attr('datetime'));
+		const artParams = {
+		  desc: $('meta[property="og:description"]').attr('content'),
+		  heroUrl: $('meta[property="og:image"]').attr('content'),
+		  videoId: $('a[data-video-id]').attr('data-video-id'),
+		};
+		$('.meta--hold, .show-for-small').remove();
+		artParams.content = $('div.article-body').html();
+		item.description = art(path.join(__dirname, 'templates/desc.art'), artParams);
+		item.media = {
+		  thumbnail: {
+			url: item.thumbnailUrl,
+		  },
+		};
+		return item;
+	  })
+	)
+  );
+
+  ctx.state.data = {
+	language: lang,
+	title,
+	description,
+	link,
+	item: items,
+  };
+};

package/lib/v2/mclaren/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:lang/:category?': ['Bubbu0129'],
+};

package/lib/v2/mclaren/radar.js

@@ -0,0 +1,26 @@
+module.exports = {
+    'mclaren.com': {
+        _name: 'Mclaren Racing',
+        www: [{
+            title: 'Mclaren Racing',
+            docs: 'https://docs.rsshub.app/new-media.html#mai-kai-lun-sai-che',
+            source: '/racing/articles/',
+            target: '/mclaren/en/all'
+            },
+        ],
+        cn: [{
+            title: '迈凯伦赛车',
+            docs: 'https://docs.rsshub.app/new-media.html#mai-kai-lun-sai-che',
+            source: '/racing/articles/',
+            target: '/mclaren/zh/all'
+            },
+        ],
+        es: [{
+            title: 'Mclaren Racing',
+            docs: 'https://docs.rsshub.app/new-media.html#mai-kai-lun-sai-che',
+            source: '/racing/articles/',
+            target: '/mclaren/es/all'
+            },
+        ],
+    },
+};

package/lib/v2/mclaren/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:lang/:category?', require('./index'));
+};

package/lib/v2/mclaren/templates/desc.art

@@ -0,0 +1,6 @@
+<p>{{ desc }}</p><br>
+<center><img src="{{ heroUrl }}"></center><br>
+{{ if videoId }}
+<center><iframe id="ytplayer" type="text/html" width="640" height="360" src="https://www.youtube-nocookie.com/embed/{{ videoId }}" frameborder="0" allowfullscreen></iframe></center><br>
+{{ /if }}
+{{@ content }}

package/lib/v2/minecraft/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/version': ['TheresaQWQ'],
+};

package/lib/v2/minecraft/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'minecraft.net': {
+        _name: '我的世界',
+        '.': [
+            {
+                title: 'Java 版游戏更新',
+                docs: 'https://docs.rsshub.app/game.html#minecraft',
+                source: ['/'],
+                target: '/minecraft/version',
+            },
+        ],
+    },
+};

package/lib/v2/minecraft/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/version', require('./version'));
+};

package/lib/{routes → v2}/minecraft/version.js

@@ -28,7 +28,7 @@ module.exports = async (ctx) => {
             description: `${item.id} ${typeMap[item.type] || ''}更新`,
             pubDate: new Date(item.releaseTime).toUTCString(),
             link: `https://www.minecraft.net`,
-            guid: item.url,
+            guid: item.id + item.type,
         })),
     };
 };

package/lib/v2/mingpao/index.js

@@ -5,6 +5,17 @@ const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
 
+const renderFanBox = (media) =>
+    art(path.join(__dirname, 'templates/fancybox.art'), {
+        media,
+    });
+
+const renderDesc = (media, desc) =>
+    art(path.join(__dirname, 'templates/description.art'), {
+        media: renderFanBox(media),
+        desc,
+    });
+
 module.exports = async (ctx) => {
     const type = ctx.params.type ?? 'ins';
     const category = ctx.params.category ?? (type === 'ins' ? 'all' : 's00001');
@@ -12,23 +23,10 @@ module.exports = async (ctx) => {
 
     const feed = await parser.parseURL(link);
 
-    const renderFanBox = (pics) =>
-        art(path.join(__dirname, 'templates/fancybox.art'), {
-            pics,
-        });
-
-    const renderDesc = (pics, desc) =>
-        art(path.join(__dirname, 'templates/description.art'), {
-            pics: renderFanBox(pics),
-            desc,
-        });
-
     const items = await Promise.all(
         feed.items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const response = await got({
-                    method: 'get',
-                    url: item.link,
+                const response = await got(item.link, {
                     headers: {
                         Referer: 'https://news.mingpao.com/',
                     },
@@ -48,15 +46,21 @@ module.exports = async (ctx) => {
                 item.category = item.categories;
 
                 // fix fancybox image
-                const fancybox = fancyboxImg
-                    .map((_, e) => {
-                        e = $(e);
-                        return {
-                            href: e.attr('href'),
-                            title: e.attr('title'),
-                        };
-                    })
-                    .get();
+                const fancybox = fancyboxImg.toArray().map((e) => {
+                    e = $(e);
+                    const href = new URL(e.attr('href'));
+                    let video;
+                    if (href.hostname === 'videop.mingpao.com') {
+                        video = new URL(href.searchParams.get('file'));
+                        video.hostname = 'cfrvideo.mingpao.com'; // use cloudflare cdn
+                        video = video.href;
+                    }
+                    return {
+                        href: href.href,
+                        title: e.attr('title'),
+                        video,
+                    };
+                });
 
                 // remove unwanted key value
                 delete item.categories;
@@ -65,7 +69,7 @@ module.exports = async (ctx) => {
                 delete item.creator;
                 delete item.isoDate;
 
-                item.description = renderDesc(fancybox, $('.txt4').html() ?? $('div.article_content.line_1_5em').html());
+                item.description = renderDesc(fancybox, $('.txt4').html() ?? $('.article_content.line_1_5em').html());
                 item.pubDate = parseDate(item.pubDate);
                 item.guid = item.link.indexOf('?') > 0 ? item.link : item.link.substring(0, item.link.lastIndexOf('/'));
 
@@ -82,13 +86,4 @@ module.exports = async (ctx) => {
         image: feed.image.url,
         language: feed.language,
     };
-
-    ctx.state.json = {
-        title: feed.title,
-        link: feed.link,
-        description: feed.description,
-        item: items,
-        image: feed.image.url,
-        language: feed.language,
-    };
 };

package/lib/v2/mingpao/templates/description.art

@@ -1,2 +1,2 @@
-{{@ pics }}
+{{@ media }}
 {{@ desc }}

package/lib/v2/mingpao/templates/fancybox.art

@@ -1,3 +1,7 @@
-{{ each pics }}
-    <figure><img src="{{ $value.href }}" alt="{{ $value.title }}"><figcaption>{{ $value.title }}</figcaption></figure>
+{{ each media }}
+    {{ if $value.video }}
+        <video controls preload="none" poster="{{ $value.video.replace('.mp4', '.jpg') }}"><source src="{{ $value.video }}" type="video/mp4"></video>
+    {{ else }}
+        <figure><img src="{{ $value.href }}" alt="{{ $value.title }}"><figcaption>{{ $value.title }}</figcaption></figure>
+    {{ /if }}
 {{ /each }}

package/lib/v2/mirror/index.js

@@ -4,10 +4,13 @@ const md = require('markdown-it')({
     html: true,
     linkify: true,
 });
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
-
+    if (!id.endsWith('.eth') && !isValidHost(id)) {
+        throw Error('Invalid id');
+    }
     const rootUrl = 'https://mirror.xyz';
     const currentUrl = id.endsWith('.eth') ? `${rootUrl}/${id}` : `https://${id}.mirror.xyz`;
 

package/lib/v2/mixcloud/index.js

@@ -0,0 +1,130 @@
+const got = require('@/utils/got');
+const CryptoJS = require('crypto-js');
+const { parseDate } = require('@/utils/parse-date');
+const { queries } = require('./queries');
+
+module.exports = async (ctx) => {
+    const host = 'https://www.mixcloud.com';
+    const imageBaseURL = 'https://thumbnailer.mixcloud.com/unsafe/480x480/';
+    const headers = {
+        Referer: host,
+        'Content-Type': 'application/json',
+        'X-Requested-With': 'XMLHttpRequest',
+    };
+
+    const type = ctx.params.type ?? 'uploads';
+    if (!['stream', 'uploads', 'favorites', 'listens'].includes(type)) {
+        throw Error(`Invalid type: ${type}`);
+    }
+    const username = ctx.params.username;
+
+    const config = {
+        stream: { name: 'Stream', node: 'stream' },
+        uploads: { name: 'Shows', node: 'uploads' },
+        favorites: { name: 'Favorites', node: 'favorites' },
+        listens: { name: 'History', node: 'listeningHistory' },
+    };
+    const payloads = {
+        stream: {
+            query: queries.stream.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+            },
+        },
+        uploads: {
+            query: queries.uploads.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+                orderBy: 'LATEST',
+                audioTypes: ['SHOW'],
+            },
+        },
+        favorites: {
+            query: queries.favorites.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+            },
+        },
+        listens: {
+            query: queries.listens.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+            },
+        },
+        profile: {
+            query: queries.profile.query,
+            variables: {
+                lookup: {
+                    username: ctx.params.username,
+                },
+            },
+        },
+    };
+
+    const profile = (
+        await got({
+            method: 'post',
+            url: `${host}/graphql`,
+            json: payloads.profile,
+            headers,
+        })
+    ).data.data;
+
+    const biog = profile.user.biog;
+    const image = `${imageBaseURL}${profile.user.picture.urlRoot}`;
+
+    const data = (
+        await got({
+            method: 'post',
+            url: `${host}/graphql`,
+            json: payloads[type],
+            headers,
+        })
+    ).data.data;
+
+    // https://github.com/ytdl-org/youtube-dl/blob/f1487d4fca40fd37d735753e24a7bae53a1b1513/youtube_dl/extractor/mixcloud.py#L72-L79
+    const decryptionKey = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD';
+    const decryptXorCipher = (key, cipherText) => {
+        const cipher = CryptoJS.enc.Base64.parse(cipherText);
+        const decrypted = cipher.toString(CryptoJS.enc.Utf8);
+        let result = '';
+        for (let i = 0; i < decrypted.length; i++) {
+            result += String.fromCharCode(decrypted.charCodeAt(i) ^ key.charCodeAt(i % key.length));
+        }
+        return result;
+    };
+
+    const items = data.user[config[type].node].edges.map((edge) => {
+        const item = type === 'listens' ? edge.node.cloudcast : edge.node;
+        return {
+            title: item.name,
+            author: item.owner.displayName,
+            description: item.description.replace(/\n/g, '<br>'),
+            pubDate: parseDate(item.publishDate),
+            guid: Buffer.from(item.id, 'base64').toString('utf8'),
+            link: `${host}/${username}/${item.slug}`,
+            itunes_item_image: `${imageBaseURL}${item.picture.urlRoot}`,
+            itunes_duration: item.audioLength,
+            enclosure_url: decryptXorCipher(decryptionKey, item.streamInfo.url),
+            enclosure_type: 'audio/x-m4a',
+            upvotes: item.favorites.totalCount,
+        };
+    });
+
+    ctx.state.data = {
+        title: `Mixcloud - ${data.user.displayName}'s ${config[type].name}`,
+        description: biog.replace(/\n/g, '<br>'),
+        itunes_author: data.user.displayName,
+        image,
+        link: `${host}/${username}/${type}/`,
+        item: items,
+    };
+};

package/lib/v2/mixcloud/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:username/:type?': ['Misaka13514'],
+};