rsshub 1.0.0-master.fc5a2f6 → 1.0.0-master.fc648b5

package/lib/v2/cscse/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/tzgg', require('./tzgg'));
+};

package/lib/v2/cscse/tzgg.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.cscse.edu.cn';
+    const currentUrl = `${rootUrl}/cscse/index/tzgg/index.html`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.news_list li a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href');
+
+            return {
+                title: item.text(),
+                link: `${/^http/.test(link) ? '' : rootUrl}${item.attr('href')}`,
+                pubDate: parseDate(item.next().text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('.zoom').html();
+                item.author = content('meta[name="Author"]').attr('content');
+                item.category = content('meta[name="Keywords"]').attr('content')?.split(',');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '中国留学网 - 通知公告',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/dgjyw/index.js

@@ -0,0 +1,67 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const params = ctx.path;
+
+    const rootUrl = 'https://www.dgjyw.com';
+    const currentUrl = `${rootUrl}${params === '/' ? '/tz' : params}.htm`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+        https: {
+            rejectUnauthorized: false,
+        },
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('div.text-list ul li a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href');
+
+            return {
+                title: item.text(),
+                link: `${/^http/.test(link) ? '' : `${rootUrl}/`}${link}`,
+                pubDate: parseDate(item.next().text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (/dgjyw\.com/.test(item.link)) {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                        https: {
+                            rejectUnauthorized: false,
+                        },
+                    });
+
+                    const content = cheerio.load(detailResponse.data);
+
+                    content('.cont-tit').remove();
+                    content('.art-body').html(content('.v_news_content').html());
+
+                    item.pubDate = timezone(parseDate(content('meta[name="PubDate"]').attr('content')), +8);
+                    item.description = content('form[name="_newscontent_fromname"]').html();
+                }
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/dgjyw/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category?': ['nczitzk'],
+};

package/lib/v2/dgjyw/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'dgjyw.com': {
+        _name: '东莞教研网',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/other.html#dong-guan-jia-yan-wang-fen-lei',
+                source: ['/'],
+                target: (params, url) => `/dgjyw/${new URL(url).toString().match(/dgjyw\.com\/(.*)\.htm$/)[1]}`,
+            },
+        ],
+    },
+};

package/lib/v2/dgjyw/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get(/([\w\d/-]+)?/, require('./index'));
+};

package/lib/v2/domp4/detail.js

@@ -0,0 +1,67 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { decodeCipherText, composeMagnetUrl } = require('./utils');
+
+const baseUrl = 'https://www.domp4.cc';
+
+function getItemList($, detailUrl) {
+    const encoded = $('.article script[type]')
+        .text()
+        .match(/return p}\('(.*)',(\d+),(\d+),'(.*)'.split\(/);
+    const data = JSON.parse(
+        decodeCipherText(encoded[1], encoded[2], encoded[3], encoded[4].split('|'), 0, {})
+            .match(/var down_urls=\\'(.*)\\'/)[1]
+            .replace(/\\\\"/g, '"')
+            .replace(/\\\\\\/g, '')
+    );
+    const { downurls } = data.Data[0];
+    return downurls.map((item) => {
+        const [title, magnet] = item.split('$');
+        return {
+            enclosure_url: composeMagnetUrl(magnet),
+            enclosure_length: '',
+            enclosure_type: 'application/x-bittorrent',
+            title,
+            link: detailUrl,
+        };
+    });
+}
+
+function getMetaInfo($) {
+    const title = $('.article-header .text p').first().find('span').text();
+    const cover = $('.article-header .pic img').attr('src');
+    const description = $('.article-related.info p').text();
+    return {
+        title,
+        cover,
+        description,
+    };
+}
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    let pureId = id;
+    let detailType = 'html';
+    // compatible for .html suffix in radar
+    if (id.endsWith('.html')) {
+        pureId = id.replace('.html', '');
+    }
+    // compatible for /detail/123.html && /html/xxx.html
+    if (/^\d+$/.test(pureId)) {
+        detailType = 'detail';
+    }
+    const detailUrl = `${baseUrl}/${detailType}/${pureId}.html`;
+
+    const res = await got(detailUrl);
+    const $ = cheerio.load(res.data);
+    const list = getItemList($, detailUrl);
+    const meta = getMetaInfo($);
+
+    ctx.state.data = {
+        link: detailUrl,
+        title: meta.title || 'domp4电影 - 详情',
+        image: meta.cover,
+        description: meta.description,
+        item: list,
+    };
+};

package/lib/v2/domp4/latest.js

@@ -0,0 +1,31 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const baseUrl = 'https://www.domp4.cc';
+const latestUrl = 'https://www.domp4.cc/custom/update.html';
+
+function getItemList($, type) {
+    const list = $(`#${type} .list-group-item`)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').text(),
+                link: `${baseUrl}${item.find('a').attr('href')}`,
+            };
+        });
+    return list;
+}
+
+module.exports = async (ctx) => {
+    const { type = 'vod' } = ctx.params;
+    const res = await got.get(latestUrl);
+    const $ = cheerio.load(res.data);
+    const list = getItemList($, type);
+
+    ctx.state.data = {
+        link: latestUrl,
+        title: 'domp4电影',
+        item: list,
+    };
+};

package/lib/v2/domp4/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/detail/:id': ['savokiss'],
+    '/latest/:type?': ['savokiss'],
+};

package/lib/v2/domp4/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'domp4.cc': {
+        _name: 'domp4电影',
+        '.': [
+            {
+                title: '最近更新',
+                docs: 'https://docs.rsshub.app/multimedia.html#domp4-ying-shi',
+                source: ['/', '/custom/update.html'],
+                target: '/domp4/latest/:type?',
+            },
+            {
+                title: '剧集订阅',
+                docs: 'https://docs.rsshub.app/multimedia.html#domp4-ying-shi',
+                source: '/html/:id',
+                target: '/domp4/detail/:id',
+            },
+            {
+                title: '剧集订阅',
+                docs: 'https://docs.rsshub.app/multimedia.html#domp4-ying-shi',
+                source: '/detail/:id',
+                target: '/domp4/detail/:id',
+            },
+        ],
+    },
+};

package/lib/v2/domp4/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/detail/:id', require('./detail'));
+    router.get('/latest/:type?', require('./latest'));
+};

package/lib/v2/domp4/utils.js

@@ -0,0 +1,72 @@
+/**
+ * trackers from https://www.domp4.cc/Style/2020/js/base.js?v=2
+ */
+const magnetTrackers = [
+    'https://tracker.iriseden.fr:443/announce',
+    'https://tr.highstar.shop:443/announce',
+    'https://tr.fuckbitcoin.xyz:443/announce',
+    'https://tr.doogh.club:443/announce',
+    'https://tr.burnabyhighstar.com:443/announce',
+    'https://t.btcland.xyz:443/announce',
+    'http://vps02.net.orel.ru:80/announce',
+    'https://tracker.kuroy.me:443/announce',
+    'http://tr.cili001.com:8070/announce',
+    'http://t.overflow.biz:6969/announce',
+    'http://t.nyaatracker.com:80/announce',
+    'http://open.acgnxtracker.com:80/announce',
+    'http://nyaa.tracker.wf:7777/announce',
+    'http://home.yxgz.vip:6969/announce',
+    'http://buny.uk:6969/announce',
+    'https://tracker.tamersunion.org:443/announce',
+    'https://tracker.nanoha.org:443/announce',
+    'https://tracker.loligirl.cn:443/announce',
+    'udp://bubu.mapfactor.com:6969/announce',
+    'http://share.camoe.cn:8080/announce',
+    'udp://movies.zsw.ca:6969/announce',
+    'udp://ipv4.tracker.harry.lu:80/announce',
+    'udp://tracker.sylphix.com:6969/announce',
+    'http://95.216.22.207:9001/announce',
+];
+
+/**
+ * compose magnet url with trackers
+ */
+function composeMagnetUrl(magnet, trackers = magnetTrackers) {
+    return `${magnet}&tr=${trackers.join('&tr=')}`;
+}
+
+/**
+ * method for generating magnet url
+ * from the detail page source
+ */
+function decodeCipherText(p, a, c, k, e, d) {
+    e = function (c) {
+        return (c < a ? '' : e(parseInt(c / a))) + ((c = c % a) > 35 ? String.fromCharCode(c + 29) : c.toString(36));
+    };
+    if (!''.replace(/^/, String)) {
+        while (c--) {
+            d[e(c)] = k[c] || e(c);
+        }
+        k = [
+            function (e) {
+                return d[e];
+            },
+        ];
+        e = function () {
+            return '\\w+';
+        };
+        c = 1;
+    }
+    while (c--) {
+        if (k[c]) {
+            p = p.replace(new RegExp('\\b' + e(c) + '\\b', 'g'), k[c]);
+        }
+    }
+    return p;
+}
+
+module.exports = {
+    magnetTrackers,
+    composeMagnetUrl,
+    decodeCipherText,
+};

package/lib/v2/douyin/hashtag.js

@@ -1,51 +1,9 @@
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
-const path = require('path');
 const config = require('@/config').value;
 const { fallback, queryToBoolean } = require('@/utils/readable-social');
-
-const templates = {
-    desc: path.join(__dirname, 'templates/desc.art'),
-    cover: path.join(__dirname, 'templates/cover.art'),
-    embed: path.join(__dirname, 'templates/embed.art'),
-    iframe: path.join(__dirname, 'templates/iframe.art'),
-};
-
-const resolveUrl = (url, tls = true, forceResolve = false) => {
-    if (!url) {
-        return '';
-    }
-    if (url.startsWith('//')) {
-        return (tls ? 'https:' : 'http:') + url;
-    }
-    if (forceResolve && !url.match(/^https?:\/\//)) {
-        return (tls ? 'https://' : 'http://') + url;
-    }
-    return url;
-};
-
-const proxyVideo = (url, proxy) => {
-    if (!(url && proxy)) {
-        return url + '';
-    }
-    if (proxy.includes('?')) {
-        if (!proxy.endsWith('=')) {
-            proxy += '=';
-        }
-        return proxy + encodeURIComponent(url);
-    } else {
-        if (!proxy.endsWith('/')) {
-            proxy += '/';
-        }
-        return proxy + url;
-    }
-};
-
-const getOriginAvatar = (url) =>
-    resolveUrl(url)
-        .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
-        .replace(/~\w+_\d+x\d+/g, '');
+const { templates, resolveUrl, proxyVideo, getOriginAvatar } = require('./utils');
 
 module.exports = async (ctx) => {
     const { cid } = ctx.params;
@@ -64,6 +22,10 @@ module.exports = async (ctx) => {
         async () => {
             const browser = await require('@/utils/puppeteer')();
             const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' ? request.continue() : request.abort();
+            });
             await page.goto(tagUrl, {
                 waitUntil: 'domcontentloaded',
             });

package/lib/v2/douyin/live.js

@@ -0,0 +1,68 @@
+const config = require('@/config').value;
+const { getOriginAvatar } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { rid } = ctx.params;
+    if (isNaN(rid)) {
+        throw Error('Invalid room ID. Room ID should be a number.');
+    }
+
+    const pageUrl = `https://live.douyin.com/${rid}`;
+
+    const renderData = await ctx.cache.tryGet(
+        `douyin:live:${rid}`,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' ? request.continue() : request.abort();
+            });
+            await page.goto(pageUrl, {
+                waitUntil: 'domcontentloaded',
+            });
+            const data = await page.evaluate(() => renderData);
+            browser.close();
+
+            return data;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    if (renderData.routeInitialProps.status_code !== '0') {
+        throw `Status code ${renderData.routeInitialProps.status_code}: ${renderData.routeInitialProps.error.title}`;
+    }
+
+    const roomInfo = renderData.initialState.roomStore.roomInfo;
+    const nickname = roomInfo.anchor.nickname;
+    const userAvatar = roomInfo.anchor.avatar_thumb.url_list[0];
+
+    const items = [];
+    if (roomInfo.roomId) {
+        if (roomInfo.room.status === 2) {
+            items.push({
+                title: `开播：${roomInfo.room.title}`,
+                link: pageUrl,
+                author: nickname,
+                guid: roomInfo.roomId, // roomId is unique for each live event
+            });
+        } else if (roomInfo.room.status === 4) {
+            items.push({
+                title: `当前直播已结束，期待下一场：${roomInfo.room.title}`,
+                link: `https://www.douyin.com/user/${roomInfo.anchor.sec_uid}`,
+                author: nickname,
+                guid: roomInfo.roomId,
+            });
+        }
+    }
+
+    ctx.state.data = {
+        title: `${nickname}的抖音直播间 - 抖音直播`,
+        description: `欢迎来到${nickname}的抖音直播间，${nickname}与大家一起记录美好生活 - 抖音直播`,
+        image: getOriginAvatar(userAvatar),
+        link: pageUrl,
+        item: items,
+        allowEmpty: true,
+    };
+};

package/lib/v2/douyin/maintainer.js

@@ -1,3 +1,5 @@
 module.exports = {
+    '/hashtag/:cid/:routeParams?': ['TonyRL'],
+    '/live/:rid': ['TonyRL'],
     '/user/:uid/:routeParams?': ['Max-Tortoise', 'Rongronggg9'],
 };

package/lib/v2/douyin/radar.js

@@ -15,5 +15,13 @@ module.exports = {
                 target: '/douyin/user/:uid',
             },
         ],
+        live: [
+            {
+                title: '直播间开播',
+                docs: 'https://docs.rsshub.app/live.html#dou-yin-zhi-bo',
+                source: '/:rid',
+                target: '/douyin/live/:rid',
+            },
+        ],
     },
 };

package/lib/v2/douyin/router.js

@@ -1,4 +1,5 @@
 module.exports = function (router) {
     router.get('/hashtag/:cid/:routeParams?', require('./hashtag'));
+    router.get('/live/:rid', require('./live'));
     router.get('/user/:uid/:routeParams?', require('./user'));
 };

package/lib/v2/douyin/user.js

@@ -1,51 +1,9 @@
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
-const path = require('path');
 const config = require('@/config').value;
 const { fallback, queryToBoolean } = require('@/utils/readable-social');
-
-const templates = {
-    desc: path.join(__dirname, 'templates/desc.art'),
-    cover: path.join(__dirname, 'templates/cover.art'),
-    embed: path.join(__dirname, 'templates/embed.art'),
-    iframe: path.join(__dirname, 'templates/iframe.art'),
-};
-
-const resolveUrl = (url, tls = true, forceResolve = false) => {
-    if (!url) {
-        return '';
-    }
-    if (url.startsWith('//')) {
-        return (tls ? 'https:' : 'http:') + url;
-    }
-    if (forceResolve && !url.match(/^https?:\/\//)) {
-        return (tls ? 'https://' : 'http://') + url;
-    }
-    return url;
-};
-
-const proxyVideo = (url, proxy) => {
-    if (!(url && proxy)) {
-        return url + '';
-    }
-    if (proxy.includes('?')) {
-        if (!proxy.endsWith('=')) {
-            proxy += '=';
-        }
-        return proxy + encodeURIComponent(url);
-    } else {
-        if (!proxy.endsWith('/')) {
-            proxy += '/';
-        }
-        return proxy + url;
-    }
-};
-
-const getOriginAvatar = (url) =>
-    resolveUrl(url)
-        .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
-        .replace(/~\w+_\d+x\d+/g, '');
+const { templates, resolveUrl, proxyVideo, getOriginAvatar } = require('./utils');
 
 module.exports = async (ctx) => {
     const uid = ctx.params.uid;
@@ -64,6 +22,10 @@ module.exports = async (ctx) => {
         async () => {
             const browser = await require('@/utils/puppeteer')();
             const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' ? request.continue() : request.abort();
+            });
             await page.goto(pageUrl, {
                 waitUntil: 'domcontentloaded',
             });

package/lib/v2/douyin/utils.js

@@ -0,0 +1,50 @@
+const path = require('path');
+
+const templates = {
+    desc: path.join(__dirname, 'templates/desc.art'),
+    cover: path.join(__dirname, 'templates/cover.art'),
+    embed: path.join(__dirname, 'templates/embed.art'),
+    iframe: path.join(__dirname, 'templates/iframe.art'),
+};
+
+const resolveUrl = (url, tls = true, forceResolve = false) => {
+    if (!url) {
+        return '';
+    }
+    if (url.startsWith('//')) {
+        return (tls ? 'https:' : 'http:') + url;
+    }
+    if (forceResolve && !url.match(/^https?:\/\//)) {
+        return (tls ? 'https://' : 'http://') + url;
+    }
+    return url;
+};
+
+const proxyVideo = (url, proxy) => {
+    if (!(url && proxy)) {
+        return url + '';
+    }
+    if (proxy.includes('?')) {
+        if (!proxy.endsWith('=')) {
+            proxy += '=';
+        }
+        return proxy + encodeURIComponent(url);
+    } else {
+        if (!proxy.endsWith('/')) {
+            proxy += '/';
+        }
+        return proxy + url;
+    }
+};
+
+const getOriginAvatar = (url) =>
+    resolveUrl(url)
+        .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
+        .replace(/~\w+_\d+x\d+/g, '');
+
+module.exports = {
+    templates,
+    resolveUrl,
+    proxyVideo,
+    getOriginAvatar,
+};

package/lib/v2/economist/full.js

@@ -10,7 +10,7 @@ const getArticleDetail = async (link, ctx) =>
         const nextData = JSON.parse($('head script[type="application/ld+json"]').text());
 
         const article = $('#new-article-template > div > div.css-wvfzu8 > div.css-mticar').html() + $('#new-article-template > div > div.css-wvfzu8 > div > div > section').html();
-        const categories = nextData.keywords.map((k) => k);
+        const categories = nextData.keywords?.map((k) => k);
 
         return { article, categories };
     });
@@ -21,14 +21,16 @@ module.exports = async (ctx) => {
 
     const items = await Promise.all(
         feed.items.map(async (item) => {
-            const itemDetails = await getArticleDetail(item.link, ctx);
+            const path = item.link.slice(item.link.lastIndexOf('/') + 1);
+            const isNotCollection = !/^\d{4}-\d{2}-\d{2}$/.test(path);
+            const itemDetails = isNotCollection ? await getArticleDetail(item.link, ctx) : null;
             return {
                 title: item.title,
-                description: itemDetails.article,
+                description: isNotCollection ? itemDetails.article : item.content,
                 link: item.link,
                 guid: item.guid,
                 pubDate: item.pubDate,
-                category: itemDetails.categories,
+                category: isNotCollection ? itemDetails.categories : null,
             };
         })
     );