rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f812e91

package/lib/v2/qdaily/index.js

@@ -1,87 +0,0 @@
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-const { parseDate } = require('@/utils/parse-date');
-const { art } = require('@/utils/render');
-const path = require('path');
-
-module.exports = async (ctx) => {
-    const baseUrl = 'http://www.qdaily.com';
-    const { type, id } = ctx.params;
-
-    const typeMap = {
-        tag: {
-            type: 'tags',
-            apiPath: 'tags/tagmore',
-            apiSuffix: '.json',
-        },
-        category: {
-            type: 'categories',
-            apiPath: 'categories/categorymore',
-            apiSuffix: '.json',
-        },
-        column: {
-            type: 'special_columns',
-            apiPath: 'special_columns/show_more',
-            apiSuffix: '',
-        },
-    };
-
-    const url = `${baseUrl}/${typeMap[type].type}/${id}.html`;
-
-    const res = await got(url);
-    const $ = cheerio.load(res.data);
-    const lastKey = $('.packery-container').attr('data-lastkey');
-
-    const { data } = await got(`${baseUrl}/${typeMap[type].apiPath}/${id}/${lastKey}${typeMap[type].apiSuffix}`);
-
-    const list = data.data.feeds.map((item) => ({
-        title: item.post.title,
-        description: item.post.description,
-        pubDate: parseDate(item.post.publish_time),
-        link: `${baseUrl}/articles/${item.post.id}.html`,
-        category: item.post.category.title,
-        image: item.image,
-    }));
-
-    const out = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const res = await got(item.link);
-                const $ = cheerio.load(res.data);
-
-                $('.author-share, .embed-mask, .lazylad, .lazyloa, .lazylod, .lazylaad, .lazylood,  .lazyloadd').remove();
-                $('.article-detail-bd')
-                    .find('img')
-                    .each((_, img) => {
-                        if (img.attribs['data-src']) {
-                            img.attribs.src = img.attribs['data-src'].split('-w600')[0];
-                            delete img.attribs['data-src'];
-                        }
-                    });
-
-                item.description = art(path.join(__dirname, 'templates/article.art'), {
-                    image: item.image.split('?')[0],
-                    description: $('.article-detail-bd').html(),
-                });
-                item.category = [
-                    ...new Set([
-                        item.category,
-                        ...$('.tags .tag a')
-                            .toArray()
-                            .map((item) => $(item).text()),
-                    ]),
-                ];
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: $('head title').text(),
-        description: $('head meta[name="description"]').attr('content'),
-        image: 'http://www.qdaily.com/favicon.ico',
-        link: url,
-        item: out,
-    };
-};

package/lib/v2/qdaily/maintainer.js

@@ -1,3 +0,0 @@
-module.exports = {
-    '/:type/:id': ['WenhuWee', 'emdoe', 'SivaGao', 'HenryQW'],
-};

package/lib/v2/qdaily/radar.js

@@ -1,25 +0,0 @@
-module.exports = {
-    'qdaily.com': {
-        _name: '好奇心日报',
-        '.': [
-            {
-                title: '标签',
-                docs: 'https://docs.rsshub.app/routes/new-media#hao-qi-xin-ri-bao',
-                source: ['/tags/:id'],
-                target: (params) => `/qdaily/tag/${params.id.replace('.html', '')}`,
-            },
-            {
-                title: '栏目',
-                docs: 'https://docs.rsshub.app/routes/new-media#hao-qi-xin-ri-bao',
-                source: ['/special_columns/:id'],
-                target: (params) => `/qdaily/column/${params.id.replace('.html', '')}`,
-            },
-            {
-                title: '分类',
-                docs: 'https://docs.rsshub.app/routes/new-media#hao-qi-xin-ri-bao',
-                source: ['/categories/:id'],
-                target: (params) => `/qdaily/category/${params.id.replace('.html', '')}`,
-            },
-        ],
-    },
-};

package/lib/v2/qdaily/router.js

@@ -1,3 +0,0 @@
-module.exports = (router) => {
-    router.get('/:type/:id', require('./index'));
-};

package/lib/v2/qianp/maintainer.js

@@ -1,3 +0,0 @@
-module.exports = {
-    '/news/:path*': ['TonyRL'],
-};

package/lib/v2/qianp/news.js

@@ -1,48 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-
-module.exports = async (ctx) => {
-    const baseUrl = 'https://qianp.com';
-    const { path = 'news/recommend' } = ctx.params;
-    const url = `${baseUrl}/${path}/`;
-
-    const { data: response } = await got(url);
-    const $ = cheerio.load(response);
-
-    const list = $('.newslist .infor')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-            const a = item.find('a').first();
-            return {
-                title: a.attr('title'),
-                link: a.attr('href'),
-            };
-        });
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const { data: response } = await got(item.link);
-                const $ = cheerio.load(response);
-
-                item.category = [...new Set($('meta[name=keywords]').attr('content').split('，'))];
-                item.author = $('meta[name=author]').attr('content');
-                item.pubDate = parseDate($('meta[property="bytedance:published_time"]').attr('content'));
-
-                item.description = $('.news_center').html();
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: $('head title').text(),
-        description: $('meta[name="description"]').attr('content'),
-        link: url,
-        image: `${baseUrl}/favicon.ico`,
-        item: items,
-    };
-};

package/lib/v2/qianp/radar.js

@@ -1,13 +0,0 @@
-module.exports = {
-    'qianp.com': {
-        _name: '千篇网',
-        '.': [
-            {
-                title: '知识库／资讯',
-                docs: 'https://docs.rsshub.app/routes/new-media#qian-pian-wang',
-                source: ['/*path'],
-                target: (params) => (!params.path.endsWith('.html') ? `/qianp/news/${params.path}` : null),
-            },
-        ],
-    },
-};

package/lib/v2/qianp/router.js

@@ -1,3 +0,0 @@
-module.exports = (router) => {
-    router.get('/news/:path*', require('./news'));
-};

package/lib/v2/radiofrance/geopolitique.js

@@ -1,33 +0,0 @@
-const path = require('path');
-const got = require('@/utils/got');
-const parser = require('@/utils/rss-parser');
-const { art } = require('@/utils/render');
-
-module.exports = async (ctx) => {
-    const feed = await parser.parseURL('https://radiofrance-podcast.net/podcast09/rss_10009.xml');
-
-    const items = await Promise.all(
-        // only retrieve the full text for the first 3 items
-        feed.items.slice(0, 3).map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const uri = item.link.slice(item.link.indexOf('/franceinter'));
-                const apiurl = 'https://www.radiofrance.fr/api/v2.1/path?value=' + encodeURIComponent(uri);
-                const { data } = await got(apiurl);
-
-                delete item.content;
-                delete item.contentSnippet;
-
-                item.description = art(path.join(__dirname, 'templates/article.art'), data.content);
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: feed.title,
-        link: feed.link,
-        description: feed.description,
-        item: items,
-        language: feed.language,
-    };
-};

package/lib/v2/radiofrance/maintainer.js

@@ -1,3 +0,0 @@
-module.exports = {
-    '/geopolitique': ['xdu'],
-};

package/lib/v2/radiofrance/radar.js

@@ -1,13 +0,0 @@
-module.exports = {
-    'radiofrance.fr': {
-        _name: 'Radio France',
-        www: [
-            {
-                title: 'Géopolitique',
-                docs: 'https://docs.rsshub.app/routes/traditional-media#fa-guo-guang-bo-dian-tai',
-                source: ['/franceinter/podcasts/geopolitique', '/'],
-                target: '/radiofrance/geopolitique',
-            },
-        ],
-    },
-};

package/lib/v2/radiofrance/router.js

@@ -1,3 +0,0 @@
-module.exports = (router) => {
-    router.get('/geopolitique', require('./geopolitique'));
-};

package/lib/v2/radiofrance/templates/article.art

@@ -1,6 +0,0 @@
-<p>{{ standFirst }}</p>
-{{ each bodyJson }}
-{{ if $value.type === 'text' }}
-<p>{{ $value.children[0].value }}</p>
-{{ /if }}
-{{ /each }}

package/lib/v2/shuiguopai/index.js

@@ -1,80 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const timezone = require('@/utils/timezone');
-const { parseDate } = require('@/utils/parse-date');
-const { art } = require('@/utils/render');
-const path = require('path');
-
-module.exports = async (ctx) => {
-    const rootUrl = 'https://sgp3.fun';
-    const apiRootUrl = 'https://api.cbbee0.com';
-    const listUrl = `${apiRootUrl}/v1_2/homePage`;
-    const filmUrl = `${apiRootUrl}/v1_2/filmInfo`;
-
-    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 50;
-
-    const response = await got({
-        method: 'post',
-        url: listUrl,
-        json: {
-            device_id: '',
-            hm: '008-api',
-            last_page: 0,
-            length: limit,
-            ltype: 1,
-            page: 1,
-            userToken: '',
-        },
-    });
-
-    let items = response.data.data.list.map((item) => ({
-        title: item.title,
-        guid: item.library_id,
-        link: `${rootUrl}/play-details/${item.library_id}`,
-        pubDate: timezone(parseDate(item.show_time, 'MM月DD日 HH:mm'), +8),
-    }));
-
-    items = await Promise.all(
-        items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-
-                const content = cheerio.load(detailResponse.data);
-                content('iframe').remove();
-
-                const infoResponse = await got({
-                    method: 'post',
-                    url: filmUrl,
-                    json: {
-                        device_id: '',
-                        film_id: detailResponse.data.match(/film_id:"([\d,]+)",/)[1],
-                        hm: '008-api',
-                        userToken: '',
-                    },
-                });
-
-                const data = infoResponse.data.data;
-
-                const videos = data.map((d) => d.download_url);
-
-                item.category = data.flatMap((d) => d.tags.map((t) => t.tag_title));
-                item.author = data.map((d) => d.actor).join(' ');
-                item.description = art(path.join(__dirname, 'templates/description.art'), {
-                    videos,
-                    description: content('.content').html(),
-                });
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: '水果派',
-        link: rootUrl,
-        item: items,
-    };
-};

package/lib/v2/shuiguopai/maintainer.js

@@ -1,3 +0,0 @@
-module.exports = {
-    '/': ['nczitzk'],
-};

package/lib/v2/shuiguopai/radar.js

@@ -1,13 +0,0 @@
-module.exports = {
-    'shuiguopai.com': {
-        _name: '水果派',
-        '.': [
-            {
-                title: '首页',
-                docs: 'https://docs.rsshub.app/routes/new-media#shui-guo-pai-shou-ye',
-                source: ['/'],
-                target: '/shuiguopai',
-            },
-        ],
-    },
-};

package/lib/v2/shuiguopai/router.js

@@ -1,3 +0,0 @@
-module.exports = function (router) {
-    router.get('/', require('./index'));
-};

package/lib/v2/shuiguopai/templates/description.art

@@ -1,8 +0,0 @@
-{{ if videos }}
-{{ each videos video }}
-<video controls>
-<source src="{{ video }}" type="video/mp4">
-</video>
-{{ /each }}
-{{ /if }}
-{{@ description }}

package/lib/v2/taiwannews/hot.js

@@ -1,52 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
-const baseUrl = 'https://www.taiwannews.com.tw';
-
-module.exports = async (ctx) => {
-    const { lang = 'en' } = ctx.params;
-    const url = `${baseUrl}/${lang}/index`;
-    const response = await got(url);
-    const $ = cheerio.load(response.data);
-
-    const list = $('.mod_group-columns  .container-fluid .row')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-            const a = item.find('a');
-            return {
-                title: a.attr('title'),
-                link: new URL(a.attr('href'), baseUrl).href,
-                pubDate: timezone(parseDate(item.find('.entry-date span').eq(1).text(), 'YYYY/MM/DD HH:mm'), +8),
-            };
-        });
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const response = await got(item.link);
-                const $ = cheerio.load(response.data);
-
-                item.author = $('.article-author').text();
-                item.category = $('.tagcloud a')
-                    .toArray()
-                    .map((a) => $(a).attr('title'));
-
-                $('.article-head-wrapper, div[id^=div-gpt-ad-], div[class^=hidden-], footer').remove();
-                $('.container-fluid').eq(2).remove();
-
-                item.description = $('.mod_single-column').html();
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: `${$('.categories').eq(0).text()} - ${$('head title').text()}`,
-        description: $('meta[name="description"]').attr('content'),
-        link: url,
-        item: items,
-        language: lang,
-    };
-};

package/lib/v2/taiwannews/maintainer.js

@@ -1,3 +0,0 @@
-module.exports = {
-    '/hot/:lang?': ['TonyRL'],
-};

package/lib/v2/taiwannews/radar.js

@@ -1,13 +0,0 @@
-module.exports = {
-    'taiwannews.com.tw': {
-        _name: '台灣英文新聞',
-        '.': [
-            {
-                title: '最新熱門消息',
-                docs: 'https://docs.rsshub.app/routes/traditional-media#tai-wan-ying-wen-xin-wen',
-                source: '/:lang/index',
-                target: '/taiwannews/hot/:lang',
-            },
-        ],
-    },
-};

package/lib/v2/taiwannews/router.js

@@ -1,3 +0,0 @@
-module.exports = (router) => {
-    router.get('/hot/:lang?', require('./hot'));
-};

package/lib/v2/twitter/web-api/twitter-got.js

@@ -1,113 +0,0 @@
-const { CookieJar, Cookie } = require('tough-cookie');
-const { promisify } = require('util');
-const queryString = require('query-string');
-const got = require('@/utils/got');
-const config = require('@/config').value;
-const constants = require('./constants');
-
-const tokens = config.twitter.authorization && config.twitter.authorization.length ? config.twitter.authorization : constants.tokens;
-
-// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L716-L726
-const headers = {
-    authorization: undefined,
-    'x-guest-token': undefined,
-    'x-twitter-auth-type': undefined,
-    'x-twitter-client-language': 'en',
-    'x-twitter-active-user': 'yes',
-    'x-csrf-token': undefined,
-    Referer: 'https://twitter.com/',
-};
-
-let cookieJar,
-    setCookie,
-    getCookies,
-    tries = 0;
-
-const cookiedomain = 'twitter.com';
-const cookieurl = 'https://twitter.com';
-
-async function twitterGot(options) {
-    const response = await got({
-        ...options,
-        headers: { ...headers, ...(options.headers || {}) },
-        cookieJar,
-    });
-    // 更新csrfToken
-    for (const c of await getCookies(cookieurl)) {
-        if (c.key === 'ct0') {
-            headers['x-csrf-token'] = c.value;
-        }
-    }
-    return response;
-}
-
-async function resetSession() {
-    cookieJar = new CookieJar();
-    getCookies = promisify(cookieJar.getCookies.bind(cookieJar));
-    setCookie = promisify(cookieJar.setCookie.bind(cookieJar));
-    let response;
-    // auth
-    let token = tokens[tries++ % tokens.length];
-    if (!token.startsWith('Bearer')) {
-        headers.authorization = `Basic ${Buffer.from(token).toString('base64')}`;
-        response = await twitterGot({
-            url: 'https://api.twitter.com/oauth2/token',
-            method: 'POST',
-            searchParams: queryString.stringify({ grant_type: 'client_credentials' }),
-        });
-        token = `Bearer ${response.data.access_token}`;
-    }
-    headers.authorization = token;
-    // 生成csrf-token
-    const csrfToken = [...Array(16 * 2)].map(() => Math.floor(Math.random() * 16).toString(16)).join('');
-    await setCookie(new Cookie({ key: 'ct0', value: csrfToken, domain: cookiedomain, secure: false }), cookieurl);
-    headers['x-csrf-token'] = csrfToken;
-    headers['x-guest-token'] = undefined;
-    // 发起初始化请求
-    response = await twitterGot({
-        url: 'https://api.twitter.com/1.1/guest/activate.json',
-        method: 'POST',
-    });
-    // 获取guest-token
-    // TODO: OAuth2Session, 参见 https://github.com/DIYgod/RSSHub/pull/7739#discussionR655932602
-    const guestToken = response.data.guest_token;
-    headers['x-guest-token'] = guestToken;
-    await setCookie(new Cookie({ key: 'gt', value: guestToken, domain: cookiedomain, secure: false }), cookieurl);
-    // 发起第二个初始化请求, 获取_twitter_sess
-    await twitterGot({
-        url: 'https://twitter.com/i/js_inst',
-        method: 'GET',
-        searchParams: queryString.stringify({ c_name: 'ui_metrics' }),
-    });
-    return cookieJar;
-}
-
-const initSession = () => cookieJar || resetSession();
-
-async function twitterRequest(url, params, method) {
-    await initSession();
-    // 发起请求
-    const request = () =>
-        twitterGot({
-            url,
-            method,
-            searchParams: queryString.stringify(params),
-        });
-    let response;
-    try {
-        response = await request();
-    } catch (e) {
-        if (e.response.status === 403 || (e.response.status === 404 && !e.response.data)) {
-            await resetSession();
-            response = await request();
-        } else {
-            throw e;
-        }
-    }
-    if (response.data.errors) {
-        throw Error('API reports error:\n' + response.data.errors.map((e) => `${e.code}: ${e.message}`).join('\n'));
-    }
-    return response.data;
-}
-
-module.exports = twitterRequest;

package/lib/v2/twreporter/category.js

@@ -1,40 +0,0 @@
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-
-const fetch = require('./fetch_article');
-
-module.exports = async (ctx) => {
-    const baseURL = 'https://www.twreporter.org';
-    const url = baseURL + `/categories/${ctx.params.cid}`;
-    const res = await got(url);
-    const $ = cheerio.load(res.data);
-    const regexp = /^window\.__REDUX_STATE__=(.*);$/gm;
-    const raw = $('script[charset="UTF-8"]').html().replace(regexp, '$1');
-    const list = JSON.parse(raw);
-    const ids = list.entities.posts.allIds;
-
-    const links = [];
-    const titles = [];
-    const category = list.entities.posts.byId[ids[0]].categories[0].name;
-
-    for (const id of ids) {
-        links.push(baseURL + '/a/' + list.entities.posts.byId[id].slug);
-        titles.push(list.entities.posts.byId[id].title);
-    }
-
-    const out = await Promise.all(
-        links.map((item, index) => {
-            const title = titles[index];
-            return ctx.cache.tryGet(item, async () => {
-                const single = await fetch(item);
-                single.title = title;
-                return single;
-            });
-        })
-    );
-    ctx.state.data = {
-        title: `報導者 | ${category}`,
-        link: url,
-        item: out,
-    };
-};

package/lib/v2/twreporter/photography.js

@@ -1,46 +0,0 @@
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-
-const fetch = require('./fetch_article');
-
-module.exports = async (ctx) => {
-    const baseURL = 'https://www.twreporter.org';
-    const url = baseURL + `/photography`;
-    const res = await got(url);
-    const $ = cheerio.load(res.data);
-    const coverList = $('.WPJvn').get();
-    const commonList = $('.eVNsZf').get();
-
-    const coverView = await Promise.all(
-        coverList.map((item) => {
-            const $ = cheerio.load(item);
-            const address = baseURL + $('li > a').attr('href');
-            const title = $('.sc-1aojo4z-4').text();
-            return ctx.cache.tryGet(address, async () => {
-                const single = await fetch(address);
-                single.title = title;
-                return single;
-            });
-        })
-    );
-
-    const listView = await Promise.all(
-        commonList.map((item) => {
-            const $ = cheerio.load(item);
-            const address = baseURL + $('li > a').attr('href');
-            const title = $('.ii0887-4').text();
-
-            return ctx.cache.tryGet(address, async () => {
-                const single = await fetch(address);
-                single.title = title;
-                return single;
-            });
-        })
-    );
-
-    ctx.state.data = {
-        title: `報導者 | 影像`,
-        link: url,
-        item: coverView.concat(listView),
-    };
-};