rsshub 1.0.0-master.fdbb6fa → 1.0.0-master.fdc59d2

package/lib/v2/science/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'science.org': {
+        _name: 'Science Magazine',
+        '.': [
+            {
+                title: '本期刊物',
+                docs: 'https://docs.rsshub.app/journal.html#science-xi-lie',
+                source: ['/journal/:journal', '/toc/:journal/current'],
+                target: '/science/current/:journal',
+            },
+            {
+                title: '封面故事',
+                docs: 'https://docs.rsshub.app/journal.html#science-xi-lie',
+                source: ['/'],
+                target: '/science/cover',
+            },
+            {
+                title: '在线发表',
+                docs: 'https://docs.rsshub.app/journal.html#science-xi-lie',
+                source: ['/journal/:journal', '/toc/:journal/0/0'],
+                target: '/science/early/:journal',
+            },
+        ],
+    },
+};

package/lib/v2/science/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/cover', require('./cover'));
+    router.get('/current/:journal?', require('./current'));
+    router.get('/early/:journal?', require('./early'));
+};

package/lib/v2/science/templates/article.art

@@ -0,0 +1,2 @@
+{{ if abs }}{{@ abs }}{{ /if }}
+{{ if content }}<br>{{@ content }}{{ /if }}

package/lib/v2/science/templates/cover.art

@@ -0,0 +1,2 @@
+<img src="{{ coverUrl }}">
+{{@ content }}

package/lib/v2/science/utils.js

@@ -0,0 +1,64 @@
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.science.org';
+
+const fetchDesc = (list, browser, tryGet) =>
+    Promise.all(
+        list.map((item) =>
+            tryGet(item.link, async () => {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+
+                await page.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
+                });
+                await page.waitForSelector('section#frontmatter, .news-article-content, .news-article-content--featured');
+                const res = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
+
+                const $ = cheerio.load(res);
+                const abstract = $('div#abstracts').html();
+                const content = $('.news-article-content--featured').length
+                    ? $('.news-article-content--featured').html()
+                    : $('.news-article-content').length
+                    ? $('.news-article-content').html()
+                    : $('.info-panel__formats a.btn__request-access').length
+                    ? ''
+                    : $('section#bodymatter').html();
+
+                item.description = art(path.join(__dirname, 'templates/article.art'), {
+                    abs: abstract,
+                    content,
+                });
+
+                return item;
+            })
+        )
+    );
+
+const getItem = (item, $) => {
+    item = $(item);
+    return {
+        title: item.find('.article-title a').attr('title'),
+        link: `${baseUrl}${item.find('.article-title a').attr('href')}`,
+        doi: item.find('.article-title a').attr('href').replace('/doi/', ''),
+        pubDate: parseDate(item.find('.card-meta__item time').text()),
+        author: item
+            .find('.card-meta ul[title="list of authors"] li')
+            .toArray()
+            .map((author) => $(author).text())
+            .join(', '),
+    };
+};
+
+module.exports = {
+    baseUrl,
+    fetchDesc,
+    getItem,
+};

package/lib/{routes → v2}/secrss/author.js

@@ -1,17 +1,19 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const api = 'https://www.secrss.com/api/articles/group?author=';
     const author = ctx.params.author;
     const host = 'https://www.secrss.com';
-    const res = await got.get(`${api}${author}`);
+    const res = await got(`${api}${author}`);
     const dataArray = res.data.data.list;
 
     const items = dataArray.map((item) => ({
         title: item.title,
         description: item.summary,
-        pubDate: new Date(item.original_timestamp * 1000).toUTCString(),
+        pubDate: parseDate(item.original_timestamp, 'X'),
         link: `${host}${item.url}`,
+        category: item.tag,
     }));
 
     ctx.state.data = {

package/lib/{routes → v2}/secrss/category.js

@@ -1,24 +1,25 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
     const api = 'https://www.secrss.com/api/articles?tag=';
-    let type = ctx.params.category;
-    if (type === undefined) {
-        type = '';
-    }
+    const { category = '' } = ctx.params;
     const host = 'https://www.secrss.com';
-    const res = await got.get(`${api}${type}`);
+    const res = await got(`${api}${category}`);
     const dataArray = res.data.data;
 
     const items = dataArray.map((item) => ({
         title: item.title,
         description: item.summary,
-        pubDate: new Date(item.published_at + ' GMT').toUTCString(),
+        pubDate: timezone(parseDate(item.published_at), +8),
         link: `${host}/articles/${item.id}`,
+        author: item.source_author,
+        category: item.tags.map((t) => t.title),
     }));
 
     ctx.state.data = {
-        title: `安全内参-${type}`,
+        title: `安全内参-${category}`,
         link: 'https://www.secrss.com',
         item: items,
     };

package/lib/v2/secrss/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/author/:author': ['XinRoom'],
+    '/category/:category?': ['XinRoom'],
+};

package/lib/v2/secrss/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'secrss.com': {
+        _name: '安全内参',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/programming.html#an-quan-nei-can',
+                source: ['/articles', '/'],
+                target: (_, url) => `/secrss/category${new URL(url).searchParams.has('tag') ? `/${new URL(url).searchParams.get('tag')}` : ''}`,
+            },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/programming.html#an-quan-nei-can',
+                source: ['/articles', '/'],
+                target: (_, url) => `/secrss/author${new URL(url).searchParams.get('author')}`,
+            },
+        ],
+    },
+};

package/lib/v2/secrss/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/author/:author', require('./author'));
+    router.get('/category/:category?', require('./category'));
+};

package/lib/v2/segmentfault/blogs.js

@@ -0,0 +1,38 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const host = 'https://segmentfault.com';
+
+module.exports = async (ctx) => {
+    const tag = ctx.params.tag;
+    const apiURL = `https://segmentfault.com/gateway/tag/${tag}/articles?loadMoreType=pagination&initData=true&page=1&sort=newest&pageSize=30`;
+    const response = await got(apiURL);
+    const data = response.data.rows;
+
+    const list = data.map((item) => ({
+        title: item.title,
+        link: new URL(item.url, host).href,
+        author: item.user.name,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const content = cheerio.load(response.data);
+
+                item.description = content('article').html();
+                item.pubDate = parseDate(content('time').attr('datetime'));
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `segmentfault-Blogs-${tag}`,
+        link: `${host}/t/${tag}/blogs`,
+        item: items,
+    };
+};

package/lib/v2/segmentfault/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/blogs/:tag': ['shiluanzzz'],
     '/channel/:name': ['LogicJake', 'Fatpandac'],
     '/user/:name': ['leyuuu', 'Fatpandac'],
 };

package/lib/v2/segmentfault/radar.js

@@ -6,13 +6,19 @@ module.exports = {
                 title: '频道',
                 docs: 'https://docs.rsshub.app/programming.html#segmentfault',
                 source: ['/channel/:name'],
-                target: '/channel/:name',
+                target: '/segmentfault/channel/:name',
             },
             {
                 title: '用户',
                 docs: 'https://docs.rsshub.app/programming.html#segmentfault',
                 source: ['/u/:name'],
-                target: '/user/:name',
+                target: '/segmentfault/user/:name',
+            },
+            {
+                title: '博客',
+                docs: 'https://docs.rsshub.app/programming.html#segmentfault',
+                source: ['/t/:tag/blogs'],
+                target: '/segmentfault/blogs/:tag',
             },
         ],
     },

package/lib/v2/segmentfault/router.js

@@ -1,4 +1,5 @@
 module.exports = function (router) {
+    router.get('/blogs/:tag', require('./blogs'));
     router.get('/channel/:name', require('./channel'));
     router.get('/user/:name', require('./user'));
 };

package/lib/v2/sensortower/blog.js

@@ -0,0 +1,69 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const language = ctx.params.language ?? '';
+
+    const rootUrl = 'https://sensortower.com';
+    const currentUrl = `${rootUrl}${language ? `/${language}` : ''}/blog`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const data = response.data.match(/"uri":"(\/blog\/.*?)"/g);
+
+    let items = data.map((item) => ({
+        link: `${rootUrl}${language ? `/${language}` : ''}${item.match(/"(\/blog\/.*?)"/)[1]}`,
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+                const detail = JSON.parse(`{${detailResponse.data.match(/("title":.*?),"body":/)[1]}}`);
+
+                content('h1').remove();
+                content('h5').parent().remove();
+                content('div[data-testid="Text-embedded-entry-block"]').remove();
+
+                content('img').each(function () {
+                    const image = (content(this).attr('srcset') ?? content(this).attr('src')).split('?w=')[0];
+
+                    content(this).replaceWith(
+                        art(path.join(__dirname, 'templates/description.art'), {
+                            image,
+                        })
+                    );
+                });
+
+                item.title = detail.title;
+                item.author = detail.author.name;
+                item.pubDate = parseDate(detail.pubDate, 'MMMM YYYY');
+                item.category = (detail.tags?.map((t) => t.title) ?? []).concat(detail.category?.map((c) => c.title) ?? []);
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    header: content('header[data-csk-entry-type="blog"]').html(),
+                    description: content('div[data-csk-entry-type="blog"] div[data-testid="Text-root"]').html(),
+                });
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'Sensor Tower - Blog',
+        link: currentUrl,
+        item: items,
+        language: language ? language : 'en-US',
+    };
+};

package/lib/v2/sensortower/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/blog/:language?': ['nczitzk'],
+};

package/lib/v2/sensortower/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'sensortower.com': {
+        _name: 'Sensor Tower',
+        '.': [
+            {
+                title: 'Blog',
+                docs: 'https://docs.rsshub.app/new-media.html#sensor-tower-blog',
+                source: ['/blog', '/zh-CN/blog', '/ja/blog', '/ko/blog', '/'],
+                target: '/sensortower/blog',
+            },
+        ],
+    },
+};

package/lib/v2/sensortower/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/blog/:language?', require('./blog'));
+};

package/lib/v2/sensortower/templates/description.art

@@ -0,0 +1,9 @@
+{{ if image }}
+<img src="{{ image }}">
+{{ /if }}
+{{ if header }}
+{{@ header }}
+{{ /if }}
+{{ if description }}
+{{@ description }}
+{{ /if }}

package/lib/v2/sputniknews/index.js

@@ -70,6 +70,8 @@ module.exports = async (ctx) => {
 
                 const content = cheerio.load(detailResponse.data);
 
+                item.pubDate = parseDate(content('a[data-unixtime]').attr('data-unixtime') * 1000);
+
                 item.category = content('.tag__text')
                     .toArray()
                     .map((tag) => content(tag).text());
@@ -78,7 +80,6 @@ module.exports = async (ctx) => {
                 content('div[data-type="article"]').remove();
 
                 item.description = content('.article').html();
-                item.pubDate = parseDate(content('div[itemprop="datePublished"]').attr('data-unixtime'));
 
                 return item;
             })

package/lib/v2/sse/convert.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const query = ctx.params.query ?? ''; // beginDate=2018-08-18&endDate=2019-08-18&companyCode=603283&title=股份
+    const pageUrl = 'https://bond.sse.com.cn/disclosure/announ/convertible/';
+    const host = 'https://www.sse.com.cn';
+
+    const response = await got('https://query.sse.com.cn/infodisplay/queryBulletinKzzTipsNew.do', {
+        searchParams: {
+            isPagination: true,
+            'pageHelp.pageSize': 20,
+            flag: 0,
+            _: new Date().getTime(),
+            ...query.split('&').reduce((acc, cur) => {
+                const [key, value] = cur.split('=');
+                acc[key] = value;
+                return acc;
+            }, {}),
+        },
+        headers: {
+            Referer: pageUrl,
+        },
+    });
+
+    const items = response.data.result.map((item) => ({
+        title: item.title,
+        description: `${host}${item.URL}`,
+        pubDate: parseDate(item.ADDDATE),
+        link: `${host}${item.URL}`,
+        author: item.security_Code,
+    }));
+
+    ctx.state.data = {
+        title: '上证债券信息网 - 可转换公司债券公告',
+        link: pageUrl,
+        item: items,
+    };
+};

package/lib/v2/sse/disclosure.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const query = ctx.params.query ?? ''; // beginDate=2018-08-18&endDate=2020-09-01&productId=600696
+    const host = 'https://www.sse.com.cn';
+    const queries = query.split('&').reduce((acc, cur) => {
+        const [key, value] = cur.split('=');
+        acc[key] = value;
+        return acc;
+    }, {});
+    const pageUrl = `https://www.sse.com.cn/assortment/stock/list/info/announcement/index.shtml?productId=${queries.productId}`;
+
+    const response = await got('https://query.sse.com.cn/security/stock/queryCompanyBulletin.do', {
+        searchParams: {
+            isPagination: true,
+            securityType: '0101,120100,020100,020200,120200',
+            reportType: 'ALL',
+            'pageHelp.pageSize': 25,
+            'pageHelp.pageCount': 50,
+            'pageHelp.pageNo': 1,
+            'pageHelp.beginPage': 1,
+            'pageHelp.cacheSize': 1,
+            'pageHelp.endPage': 5,
+            _: new Date().getTime(),
+            ...queries,
+        },
+        headers: {
+            Referer: pageUrl,
+        },
+    });
+
+    const items = response.data.result.map((item) => ({
+        title: item.TITLE,
+        description: `${host}${item.URL}`,
+        pubDate: parseDate(item.ADDDATE),
+        link: `${host}${item.URL}`,
+        author: item.SECURITY_NAME,
+    }));
+
+    ctx.state.data = {
+        title: `上海证券交易所 - 上市公司信息 - ${items[0].author}最新公告`,
+        link: pageUrl,
+        item: items,
+    };
+};

package/lib/v2/sse/inquire.js

@@ -0,0 +1,45 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const refererUrl = 'https://www.sse.com.cn/disclosure/credibility/supervision/inquiries/';
+    const response = await got('https://query.sse.com.cn/commonSoaQuery.do', {
+        searchParams: {
+            isPagination: true,
+            'pageHelp.pageSize': 25,
+            'pageHelp.pageNo': 1,
+            'pageHelp.beginPage': 1,
+            'pageHelp.cacheSize': 1,
+            'pageHelp.endPage': 1,
+            sqlId: 'BS_KCB_GGLL',
+            siteId: 28,
+            channelId: '10743,10744,10012',
+            type: '',
+            stockcode: '',
+            extWTFL: '',
+            order: 'createTime|desc,stockcode|asc',
+            _: new Date().getTime(),
+        },
+        headers: {
+            Referer: refererUrl,
+        },
+    });
+
+    const items = response.data.result.map((item) => ({
+        title: item.extGSJC,
+        description: art(path.resolve(__dirname, 'templates/inquire.art'), {
+            item,
+        }),
+        pubDate: parseDate(item.createTime),
+        link: `https://${item.docURL}`,
+        author: item.extGSJC,
+    }));
+
+    ctx.state.data = {
+        title: '上海证券交易所 - 科创板股票审核',
+        link: refererUrl,
+        item: items,
+    };
+};

package/lib/v2/sse/lawandrules.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const slug = ctx.params.slug ?? 'latest';
+
+    const rootUrl = 'https://www.sse.com.cn';
+    const currentUrl = `${rootUrl}/lawandrules/guide/${slug.replace(/-/g, '/')}`;
+    const response = await got(currentUrl);
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('.sse_list_1 dl dd')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: `${rootUrl}${item.find('a').attr('href')}`,
+                pubDate: parseDate(item.find('span').text().trim()),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('.allZoom').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/sse/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/convert/:query?': ['kt286'],
+    '/disclosure/:query?': ['alienmao'],
+    '/inquire': ['Jeason0228'],
+    '/lawandrules/:slug?': ['nczitzk'],
+    '/renewal': ['Jeason0228'],
+};

package/lib/v2/sse/radar.js

@@ -0,0 +1,41 @@
+module.exports = {
+    'sse.com.cn': {
+        _name: '上海证券交易所',
+        bond: [
+            {
+                title: '可转换公司债券公告',
+                docs: 'https://docs.rsshub.app/finance.html#shang-hai-zheng-quan-jiao-yi-suo',
+                source: ['/disclosure/announ/convertible', '/'],
+                // target: '/sse/convert/:query',
+            },
+        ],
+        kcb: [
+            {
+                title: '科创板项目动态',
+                docs: 'https://docs.rsshub.app/finance.html#shang-hai-zheng-quan-jiao-yi-suo',
+                source: ['/home', '/'],
+                target: '/sse/renewal',
+            },
+        ],
+        www: [
+            {
+                title: '监管问询',
+                docs: 'https://docs.rsshub.app/finance.html#shang-hai-zheng-quan-jiao-yi-suo',
+                source: ['/disclosure/credibility/supervision/inquiries', '/'],
+                target: '/sse/inquire',
+            },
+            {
+                title: '上市公司信息最新公告披露',
+                docs: 'https://docs.rsshub.app/finance.html#shang-hai-zheng-quan-jiao-yi-suo',
+                source: ['/assortment/stock/list/info/announcement/index.shtml', '/'],
+                // target: '/sse/disclosure/:query'
+            },
+            {
+                title: '本所业务指南与流程',
+                docs: 'https://docs.rsshub.app/finance.html#shang-hai-zheng-quan-jiao-yi-suo',
+                source: ['/lawandrules/guide/*slug', '/'],
+                target: (params) => `/sse/lawandrules/${params.slug.replace(/\//g, '-')}`,
+            },
+        ],
+    },
+};