rsshub 1.0.0-master.f8b1c8d → 1.0.0-master.f91e29d

package/lib/v2/crossbell/notes/source.js

@@ -0,0 +1,19 @@
+const got = require('@/utils/got');
+const { getItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const source = ctx.params.source;
+
+    const response = await got('https://indexer.crossbell.io/v1/notes', {
+        searchParams: {
+            sources: source,
+            includeCharacter: true,
+        },
+    });
+
+    ctx.state.data = {
+        title: 'Crossbell Notes from ' + source,
+        link: 'https://crossbell.io/',
+        item: response.data?.list?.map((item) => getItem(item)),
+    };
+};

package/lib/v2/crossbell/notes/utils.js

@@ -0,0 +1,18 @@
+module.exports = {
+    getItem: (note) => {
+        let link = note.metadata?.content?.external_urls?.[0] ?? `https://crossbell.io/notes/${note.characterId}-${note.noteId}`;
+        if (link.startsWith('https://xn--')) {
+            link = `https://crossbell.io/notes/${note.characterId}-${note.noteId}`;
+        }
+        return {
+            title: note.metadata?.content?.title || '',
+            description: note.metadata?.content?.content,
+            link,
+            pubDate: note.metadata?.content?.publishedAt,
+            updated: note.metadata?.content?.updatedAt,
+            author: note.metadata?.content?.authors?.[0] || note.character?.metadata?.content?.name || note.character?.handle,
+            guid: `https://crossbell.io/notes/${note.characterId}-${note.noteId}`,
+            category: [...(note.metadata?.content?.sources || []), ...(note.metadata?.content?.tags || [])],
+        };
+    },
+};

package/lib/v2/crossbell/radar.js

@@ -0,0 +1,24 @@
+module.exports = {
+    'crossbell.io': {
+        _name: 'Crossbell',
+        '.': [
+            {
+                title: 'Notes',
+                docs: 'https://docs.rsshub.app/social-media.html#crossbell',
+                source: '/*',
+                target: '/crossbell/notes',
+            },
+        ],
+    },
+    'xlog.app': {
+        _name: 'xLog',
+        '.': [
+            {
+                title: 'Notes',
+                docs: 'https://docs.rsshub.app/social-media.html#crossbell',
+                source: '/*',
+                target: '/crossbell/notes/source/xlog',
+            },
+        ],
+    },
+};

package/lib/v2/crossbell/router.js

@@ -0,0 +1,6 @@
+module.exports = function (router) {
+    router.get('/notes/character/:characterId', require('./notes/character'));
+    router.get('/notes/source/:source', require('./notes/source'));
+    router.get('/notes', require('./notes/index'));
+    router.get('/feeds/following/:characterId', require('./feeds/following'));
+};

package/lib/v2/csu/career.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { unzip } = require('./utils');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://career.csu.edu.cn';
+    const link = `${baseUrl}/campus/index/category/1`;
+    const { data: response } = await got(link);
+    const $ = cheerio.load(response);
+
+    const list = $('.infoList')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').text(),
+                link: `${baseUrl}${item.find('a').attr('href')}`,
+                pubDate: timezone(parseDate(item.find('.span4').text()), 8),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                let $ = cheerio.load(response);
+
+                const zipped = $('script[type="text/javascript"]')
+                    .text()
+                    .match(/Base64\.decode\(unzip\("(.*)"\)\./)[1];
+                const { slice1, slice2 } = $('script[type="text/javascript"]')
+                    .text()
+                    .match(/"\)\.substr\((?<slice1>\d+)\)\)\.substr\((?<slice2>\d+)\)\);/).groups;
+                const unzipped = Buffer.from(unzip(zipped).slice(slice1), 'base64').toString().slice(slice2);
+
+                $ = cheerio.load(unzipped, null, false);
+                item.description = $.html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${$('.curr').text()} - ${$('head title').text()}`,
+        link,
+        item: items,
+    };
+};

package/lib/v2/csu/mail.js

@@ -1,42 +1,46 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 async function fetch(address) {
     const res = await got(address);
     const $ = cheerio.load(res.data);
-    return {
-        description: $('.tb-ct-info').html(),
-        link: address,
-        guid: address,
-    };
+    return $('.tb-ct-info').html();
 }
 
 module.exports = async (ctx) => {
-    const url = 'http://oa.csu.edu.cn/WebServer/MailBoxNew/MailList_Pub.aspx?tp=';
-    const type = ctx.params.type;
-    const link = url + type;
-    const response = await got.get(link);
-    const $ = cheerio.load(response.data);
-    const list = $('.tbl-tb tr').get().slice(1);
+    const baseUrl = 'https://oa.csu.edu.cn';
+    const { type = '01' } = ctx.params;
+    const link = `${baseUrl}/mailbox/NoAuth/MailList_Pub?tp=${type}`;
+
+    const response = await got.post(`${baseUrl}/mailbox/NoAuth/Get_MailList_Pub`, {
+        form: {
+            params: `{"XXLX":"${type}","tjnr":""}`,
+            pageSize: 1,
+            pageNo: 15,
+        },
+    });
+
+    const list = response.data.data.map((item) => ({
+        title: item.WJBT,
+        link: `${baseUrl}/mailbox/NoAuth/MailInInfo?XXLX=${type}&id=${item.NBBM}`,
+        pubDate: timezone(parseDate(item.LXSJ), 8),
+        author: item.FZDW,
+        category: item.NRFL,
+    }));
+
     const out = await Promise.all(
-        list.map((item) => {
-            const $ = cheerio.load(item);
-            const address = new URL($('a').attr('href'), url).href;
-            const title = $('a').text();
-            const author = $('td').eq(2).text();
-            const pubDate = $('td').eq(3).text();
-            return ctx.cache.tryGet(address, async () => {
-                const single = await fetch(address);
-                single.title = title;
-                single.author = author;
-                single.pubDate = parseDate(pubDate, 'YYYY/MM/DD');
-                return single;
-            });
-        })
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                item.description = await fetch(item.link);
+                return item;
+            })
+        )
     );
+
     ctx.state.data = {
-        title: $('title').text(),
+        title: `中南大学学校信箱 - ${type === '01' ? '校长信箱' : '党委信箱'}`,
         link,
         item: out,
     };

package/lib/v2/csu/maintainer.js

@@ -1,3 +1,5 @@
 module.exports = {
+    '/career': ['TonyRL'],
     '/cse/:type?': ['j1g5awi'],
+    '/mail/:type?': ['j1g5awi'],
 };

package/lib/v2/csu/radar.js

@@ -1,6 +1,14 @@
 module.exports = {
     'csu.edu.cn': {
         _name: '中南大学',
+        career: [
+            {
+                title: '就业信息网招聘信息',
+                docs: 'https://docs.rsshub.app/university.html#zhong-nan-da-xue',
+                source: ['/campus/index/category/1', '/campus', '/'],
+                target: '/csu/career',
+            },
+        ],
         cse: [
             {
                 title: '计算机学院',
@@ -13,8 +21,8 @@ module.exports = {
             {
                 title: '校长信箱',
                 docs: 'https://docs.rsshub.app/university.html#zhong-nan-da-xue',
-                source: ['/WebServer/MailBoxNew/MailList_Pub.aspx?tp=:type'],
-                target: (params) => `/csu/mail/${params.type}`,
+                source: ['/mailbox/NoAuth/MailList_Pub'],
+                target: (_, url) => `/csu/mail/${new URL(url).searchParams.get('tp')}`,
             },
         ],
     },

package/lib/v2/csu/router.js

@@ -1,5 +1,5 @@
 module.exports = function (router) {
+    router.get('/career', require('./career'));
     router.get('/cse/:type?', require('./cse'));
-    router.get('/job/:type?', require('./job'));
     router.get('/mail/:type?', require('./mail'));
 };

package/lib/v2/csu/utils.js

@@ -0,0 +1,17 @@
+const { inflateSync } = require('zlib');
+
+const unzip = (b64Data) => {
+    const strData = Buffer.from(b64Data, 'base64').toString('binary');
+    const charData = strData.split('').map((x) => x.charCodeAt(0));
+    const binData = new Uint8Array(charData);
+    const data = inflateSync(binData);
+    let ret = '';
+    for (const c of data) {
+        ret += String.fromCharCode(c);
+    }
+    return ret;
+};
+
+module.exports = {
+    unzip,
+};

package/lib/v2/curius/links.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const username = ctx.params.name;
+
+    const name_response = await got(`https://curius.app/api/users/${username}`, {
+        headers: {
+            Referer: `https://curius.app/${username}`,
+        },
+    });
+
+    const data = name_response.data;
+
+    const uid = data.user.id;
+    const name = `${data.user.firstName} ${data.user.lastName}`;
+
+    const response = await got(`https://curius.app/api/users/${uid}/links?page=0`, {
+        headers: {
+            Referer: `https://curius.app/${username}`,
+        },
+    });
+
+    const items = response.data.userSaved.map((item) => ({
+        title: item.title,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        }),
+        link: item.link,
+        pubDate: parseDate(item.createdDate),
+        guid: `curius:${username}:${item.id}`,
+    }));
+
+    ctx.state.data = {
+        title: `${name} - Curius`,
+        link: `https://curius.app/${username}`,
+        description: `${name} - Curius`,
+        allowEmpty: true,
+        item: items,
+    };
+};

package/lib/v2/curius/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/links/:name': ['Ovler-Young'],
+};

package/lib/v2/curius/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'curius.app': {
+        _name: 'Curius',
+        '.': [
+            {
+                title: '用户',
+                docs: 'https://docs.rsshub.app/social-media.html#curius',
+                source: '/:name',
+                target: '/curius/links/:name',
+            },
+        ],
+    },
+};

package/lib/v2/curius/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/links/:name', require('./links'));
+};

package/lib/v2/curius/templates/description.art

@@ -0,0 +1,16 @@
+{{ if item.metadata.full_text }}
+    原文：{{@ item.metadata.full_text.replace(/\n/gm, '<br>') }}<br><br>
+{{ /if }}
+{{ if item.comments.length }}
+    评论：{{ item.comments[0].text.substring(0, 100) }}<br><br>
+{{ /if }}
+{{ if item.highlights.length }}
+    评论：
+    {{ each item.highlights highlight }}
+        {{ if highlight.comment }}
+            {{ highlight.highlight }}<br>评论：{{ highlight.comment.text }}<br>
+        {{ else }}
+            <br>标注：{{ highlight.highlight }}<br>
+        {{ /if }}
+    {{ /each }}
+{{ /if }}

package/lib/v2/cw/author.js

@@ -0,0 +1,18 @@
+const { baseUrl, parsePage } = require('./utils');
+
+module.exports = async (ctx) => {
+    const browser = await require('@/utils/puppeteer')();
+
+    const { $, items } = await parsePage('author', browser, ctx);
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('.authorTxt').text(),
+        link: `${baseUrl}/author/${ctx.params.channel}`,
+        image: $('.authorPhoto img').attr('src') || `${baseUrl}/assets_new/img/fbshare.jpg'`,
+        language: $('meta[property="og:locale"]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/cw/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/author/:channel': ['TonyRL'],
     '/master/:channel': ['TonyRL'],
     '/sub/:channel': ['TonyRL'],
     '/today': ['TonyRL'],

package/lib/v2/cw/master.js

@@ -1,26 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const { channel } = ctx.params;
-    const pageUrl = `${baseUrl}/masterChannel.action`;
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-        searchParams: {
-            idMasterChannel: channel,
-        },
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('master', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/masterChannel.action?idMasterChannel=${ctx.params.channel}`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/radar.js

@@ -20,6 +20,12 @@ module.exports = {
                 source: ['/subchannel.action'],
                 target: (_, url) => `/cw/sub/${new URL(url).searchParams.get('idSubChannel')}`,
             },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/traditional-media.html#tian-xia-za-zhi',
+                source: ['/author/:channel'],
+                target: '/cw/author/:channel',
+            },
         ],
     },
 };

package/lib/v2/cw/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
+    router.get('/author/:channel', require('./author'));
     router.get('/master/:channel', require('./master'));
     router.get('/sub/:channel', require('./sub'));
     router.get('/today', require('./today'));

package/lib/v2/cw/sub.js

@@ -1,26 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const { channel } = ctx.params;
-    const pageUrl = `${baseUrl}/subchannel.action`;
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-        searchParams: {
-            idSubChannel: channel,
-        },
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('sub', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/subchannel.action?idSubChannel=${ctx.params.channel}`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/today.js

@@ -1,22 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const pageUrl = `${baseUrl}/today`;
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('today', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/today`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/utils.js

@@ -1,16 +1,74 @@
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-const { CookieJar } = require('tough-cookie');
-const cookieJar = new CookieJar();
-const config = require('@/config').value;
+const { getCookies, setCookies } = require('@/utils/puppeteer-utils');
+const logger = require('@/utils/logger');
+let cookie;
 
 const baseUrl = 'https://www.cw.com.tw';
 
-const got = require('@/utils/got').extend({
-    headers: {
-        'User-Agent': config.trueUA,
+const pathMap = {
+    today: {
+        pageUrl: () => '/today',
+        limit: 30,
     },
-});
+    master: {
+        pageUrl: (channel) => `/masterChannel.action?idMasterChannel=${channel}`,
+        limit: 12,
+    },
+    sub: {
+        pageUrl: (channel) => `/subchannel.action?idSubChannel=${channel}`,
+        limit: 12,
+    },
+    author: {
+        pageUrl: (channel) => `/author/${channel}`,
+        limit: 10,
+    },
+};
+
+const getCookie = async (browser, tryGet) => {
+    if (!cookie) {
+        cookie = await tryGet('cw:cookie', async () => {
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
+            logger.debug(`Requesting ${baseUrl}/user/get/cookie-bar`);
+            await page.goto(`${baseUrl}/user/get/cookie-bar`, {
+                waitUntil: 'domcontentloaded',
+            });
+            cookie = await getCookies(page);
+            await page.close();
+            return cookie;
+        });
+    }
+    return cookie;
+};
+
+const parsePage = async (path, browser, ctx) => {
+    const pageUrl = `${baseUrl}${pathMap[path].pageUrl(ctx.params.channel)}`;
+
+    const cookie = await getCookie(browser, ctx.cache.tryGet);
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await setCookies(page, cookie, 'cw.com.tw');
+    logger.debug(`Requesting ${pageUrl}`);
+    await page.goto(pageUrl, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    const response = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    const $ = cheerio.load(response);
+
+    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : pathMap[path].limit);
+    const items = await parseItems(list, browser, ctx.cache.tryGet);
+
+    return { $, items };
+};
 
 const parseList = ($, limit) =>
     $('.caption')
@@ -25,15 +83,25 @@ const parseList = ($, limit) =>
         })
         .slice(0, limit);
 
-const parseItems = (list, tryGet) =>
+const parseItems = (list, browser, tryGet) =>
     Promise.all(
         list.map((item) =>
             tryGet(item.link, async () => {
-                const { data: response } = await got(item.link, {
-                    cookieJar,
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await setCookies(page, cookie, 'cw.com.tw');
+                logger.debug(`Requesting ${item.link}`);
+                await page.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
                 });
 
+                const response = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
                 const $ = cheerio.load(response);
+
                 const meta = JSON.parse($('head script[type="application/ld+json"]').eq(0).text());
                 $('.article__head .breadcrumb, .article__head h1, .article__provideViews, .ad').remove();
                 $('img.lazyload').each((_, img) => {
@@ -56,8 +124,10 @@ const parseItems = (list, tryGet) =>
 
 module.exports = {
     baseUrl,
-    cookieJar,
-    got,
+    pathMap,
+    getCookie,
+    setCookies,
+    parsePage,
     parseList,
     parseItems,
 };

package/lib/v2/cztv/daily.js

@@ -0,0 +1,41 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const renderDesc = (item) => art(path.join(__dirname, 'templates/daily.art'), item);
+
+module.exports = async (ctx) => {
+    const url = 'http://www.cztv.com/videos/zjxwlb';
+
+    const { data: res } = await got(url);
+    const $ = cheerio.load(res);
+    const list = $('#videolistss li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const title = item.find('span.t1').text();
+            const link = item.find('input[name=data-url]').attr('value');
+
+            return {
+                title,
+                link,
+                pubDate: timezone(parseDate(item.find('span.t2').text() + ' 16:30', 'YYYY-MM-DD hh:mm'), +8),
+            };
+        });
+
+    const out = {
+        title: list[0].title,
+        link: list[0].link,
+        pubDate: list[0].pubDate,
+        description: renderDesc({ list: list.slice(1) }),
+    };
+
+    ctx.state.data = {
+        title: '浙江新闻联播-每日合集',
+        link: url,
+        item: [out],
+    };
+};

package/lib/v2/cztv/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/zjxwlb': ['yhkang'],
+    '/zjxwlb/daily': ['yhkang'],
+};