rsshub 1.0.0-master.fc91df6 → 1.0.0-master.fd40700

package/lib/{routes → v2}/zaobao/index.js

@@ -2,8 +2,8 @@ const { parseList } = require('./util');
 const baseUrl = 'https://www.zaobao.com';
 
 module.exports = async (ctx) => {
-    const type = ctx.params.type || 'realtime';
-    const section = ctx.params.section || 'china';
+    const type = ctx.params.type ?? 'realtime';
+    const section = ctx.params.section ?? 'china';
     const sectionLink = `/${type}/${section}`;
 
     const { title, resultList } = await parseList(ctx, sectionLink);

package/lib/v2/zaobao/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/interactive-graphics': ['shunf4'],
+    '/realtime/:section?': ['shunf4'],
+    '/znews/:section?': ['shunf4'],
+    '/:type/:section': ['shunf4'],
+};

package/lib/v2/zaobao/radar.js

@@ -0,0 +1,28 @@
+const radarConfig = {
+    _name: '联合早报',
+    www: [
+        {
+            title: '新闻-新加坡',
+            docs: 'https://docs.rsshub.app/traditional-media.html#lian-he-zao-bao-xin-wen',
+            source: ['/', '/news', '/news/singapore'],
+            target: '/zaobao/znews/singapore',
+        },
+        {
+            title: '新闻-中国',
+            docs: 'https://docs.rsshub.app/traditional-media.html#lian-he-zao-bao-xin-wen',
+            source: ['/', '/news', '/news/china'],
+            target: '/zaobao/znews/china',
+        },
+        {
+            title: '新闻-国际',
+            docs: 'https://docs.rsshub.app/traditional-media.html#lian-he-zao-bao-xin-wen',
+            source: ['/', '/news', '/news/world'],
+            target: '/zaobao/znews/world',
+        },
+    ],
+};
+
+module.exports = {
+    'zaobao.com': radarConfig,
+    'zaobao.com.sg': radarConfig,
+};

package/lib/{routes → v2}/zaobao/realtime.js

@@ -2,7 +2,7 @@ const { parseList } = require('./util');
 const baseUrl = 'https://www.zaobao.com';
 
 module.exports = async (ctx) => {
-    const section = ctx.params.section || 'china';
+    const section = ctx.params.section ?? 'china';
 
     let name = '中港台';
     let sectionLink = '/realtime/china';
@@ -14,7 +14,9 @@ module.exports = async (ctx) => {
         sectionLink = '/realtime/world';
     } else if (section === 'zfinance') {
         name = '财经';
-        sectionLink = '/zfinance/realtime';
+        // this is for HK version; for SG version, it's redirected to
+        // /realtime/finance
+        sectionLink = '/finance/realtime';
     }
 
     const { resultList } = await parseList(ctx, sectionLink);

package/lib/v2/zaobao/router.js

@@ -0,0 +1,6 @@
+module.exports = function (router) {
+    router.get('/interactive-graphics', require('./interactive'));
+    router.get('/realtime/:section?', require('./realtime'));
+    router.get('/znews/:section?', require('./znews'));
+    router.get('/:type?/:section?', require('./index'));
+};

package/lib/v2/zaobao/templates/zaobao.art

@@ -0,0 +1,17 @@
+{{ each imageDataArray imageData }}
+    {{ if imageData.type == 'normalHTML' }}
+        {{@ imageData.html }}
+    {{ /if }}
+    {{ if imageData.type == 'data' }}
+        <figure>
+            <img src="{{ imageData.src }}" />
+            <figcaption>
+                {{ imageData.title }}
+            </figcaption>
+        </figure>
+    {{ /if }}
+{{ /each }}
+
+{{ if articleBody }}
+    {{@ articleBody }}
+{{ /if }}

package/lib/{routes → v2}/zaobao/util.js

@@ -1,6 +1,10 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const date = require('@/utils/date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
 const baseUrl = 'https://www.zaobao.com';
 const got_ins = got.extend({
     headers: {
@@ -27,7 +31,8 @@ const parseList = async (ctx, sectionUrl) => {
     const $ = cheerio.load(response.data);
     let data = $('.article-list').find('.article-type');
     if (data.length === 0) {
-        data = $('.post-list').find('.row.list');
+        // for HK version
+        data = $('.clearfix').find('.list-block');
     }
 
     const title = $('#breadcrumbs > a')
@@ -41,7 +46,8 @@ const parseList = async (ctx, sectionUrl) => {
     const resultList = await Promise.all(
         data.toArray().map(async (item) => {
             const $item = $(item);
-            let link = $item.find('a')[0].attribs.href;
+            // addBack: for HK version
+            let link = $item.find('a').addBack('a')[0].attribs.href;
 
             let resultItem = {};
 
@@ -53,16 +59,16 @@ const parseList = async (ctx, sectionUrl) => {
                     dateNodes = $item.find('.datestamp');
                 }
                 let dateString;
-                let pubDate;
+                let pubDate = undefined;
                 if (dateNodes.length !== 0) {
                     dateString = dateNodes.text().trim();
                     const dateParts = dateString.split('/');
                     dateParts.reverse();
                     dateString = dateParts.join('-');
                     pubDate = date(dateString);
-                    pubDate = pubDate === dateString ? new Date().toUTCString() : pubDate;
-                } else {
-                    pubDate = new Date().toUTCString();
+                    // if conversion was no success, pubDate === dateString
+                    // zaobao seems always UTC+8
+                    pubDate = pubDate === dateString ? undefined : timezone(pubDate, +8);
                 }
 
                 resultItem = {
@@ -84,33 +90,64 @@ const parseList = async (ctx, sectionUrl) => {
                 const article = await got_ins.get(link);
                 const $1 = cheerio.load(article.data);
 
-                const time = $1('.date-published', '.content').slice(0, 1).text().replace('年', '-').replace('月', '-').replace('日', '').replace('发布 /', '');
+                let time = $1('.date-published', '.content').slice(0, 1).text().trim();
+                if (!time) {
+                    // for HK version
+                    time = $1('.author-info').find('.mgt10').slice(0, 1).text().trim();
+                }
+                time = time.replace('年', '-').replace('月', '-').replace('日', '').replace('发布 /', '');
 
                 $1('.overlay-microtransaction').remove();
                 $1('#video-freemium-player').remove();
                 $1('script').remove();
 
-                let descriptionNode = $1('.article-content-rawhtml');
-                if (descriptionNode.length === 0) {
-                    descriptionNode = $1('.article-content-container');
+                let articleBodyNode = $1('.article-content-rawhtml');
+                if (articleBodyNode.length === 0) {
+                    // for HK version
+                    articleBodyNode = $1('.article-body');
                 }
 
-                let description = descriptionNode.html();
+                const articleBody = articleBodyNode.html();
+
+                const imageDataArray = [];
                 if ($1('.inline-figure-img').length) {
-                    description = $1('.inline-figure-img').html() + description;
+                    // for SG version
+                    imageDataArray.push({
+                        type: 'normalHTML',
+                        html: $1('.inline-figure-img').html(),
+                    });
                 }
                 if ($1('.body-content .loadme picture img').length) {
-                    description =
-                        `<figure><img src="${$1('.body-content .loadme picture source').attr('data-srcset').split('?')[0]}" /><figcaption>${$1('.body-content .loadme picture img').attr('title')}</figcaption></figure>` + description;
+                    // Unused?
+                    imageDataArray.push({
+                        type: 'data',
+                        src: $1('.body-content .loadme picture source').attr('data-srcset'),
+                        title: $1('.body-content .loadme picture img').attr('title'),
+                    });
                 }
                 if ($1('.inline-figure-gallery').length) {
-                    description = $1('.inline-figure-gallery').html() + description;
+                    // Unused?
+                    imageDataArray.push({
+                        type: 'normalHTML',
+                        html: $1('.inline-figure-gallery').html(),
+                    });
+                }
+                if ($1('#carousel-article').length) {
+                    // for HK version
+                    imageDataArray.push({
+                        type: 'normalHTML',
+                        html: $1('#carousel-article .carousel-inner').html(),
+                    });
                 }
 
                 resultItem = {
-                    title: $1('h1', '.content').text(),
-                    description,
-                    pubDate: new Date(time).toUTCString(),
+                    // <- for SG version  -> for HK version
+                    title: $1('h1', '.content').text().trim() || $1('h1.article-title').text(),
+                    description: art(path.join(__dirname, 'templates/zaobao.art'), {
+                        articleBody,
+                        imageDataArray,
+                    }),
+                    pubDate: timezone(time, +8), // zaobao seems always UTC+8
                     link,
                 };
                 ctx.cache.set(link, JSON.stringify(resultItem));

package/lib/{routes → v2}/zaobao/znews.js

@@ -19,9 +19,6 @@ module.exports = async (ctx) => {
     } else if (section === 'sports') {
         info = '体育';
         sectionLink = '/news/sports';
-    } else if (section === 'fukan') {
-        info = '早报现在';
-        sectionLink = '/news/fukan';
     }
 
     const { resultList } = await parseList(ctx, sectionLink);

package/lib/v2/zjgtjy/index.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type === 'all' ? '' : ctx.params.type.toUpperCase();
+    const host = `https://td.zjgtjy.cn:8553/devops/noticeInfo/queryNoticeInfoList?pageSize=10&pageNumber=1&noticeType=${type}&sort=DESC`;
+
+    const response = await got({
+        method: 'get',
+        url: host,
+    }).json();
+    const data = response.data;
+
+    const items = await Promise.all(
+        data.map(async (item) => {
+            const pageUrl = `https://td.zjgtjy.cn:8553/devops/noticeInfo/queryNoticeLandContentDetails?noticeId=${item.GGID}&transactionMode=${item.JYFS}`;
+            const pageLink = `https://td.zjgtjy.cn/view/trade/announcement/detail?id=${item.GGID}&category=${item.ZYLB}&type=${item.JYFS}`;
+
+            const desc = await ctx.cache.tryGet(pageUrl, async () => {
+                let desc = await got({
+                    method: 'get',
+                    url: pageUrl,
+                }).json();
+                desc = desc.queryNoticeContent.GGNR;
+
+                desc = desc
+                    .replace(/&lt;/g, '<')
+                    .replace(/&gt;/g, '>')
+                    .replace(/&quot;/g, '"');
+                return desc;
+            });
+
+            return {
+                title: item.GGMC,
+                description: desc,
+                link: pageLink,
+                pubDate: item.GGFBSJ,
+            };
+        })
+    );
+
+    ctx.state.data = {
+        title: '浙江土地使用权挂牌公告',
+        link: host,
+        item: items,
+    };
+};

package/lib/v2/zjgtjy/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:type?': ['Fatpandac'],
+};

package/lib/v2/zjgtjy/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'zjgtjy.cn': {
+        _name: '浙江省土地使用权网上交易系统',
+        '.': [
+            {
+                title: '全部更新',
+                docs: 'https://docs.rsshub.app/government.html',
+                source: '/',
+                target: '/zjgtjy/all',
+            },
+            {
+                title: '挂牌公告',
+                docs: 'https://docs.rsshub.app/government.html',
+                source: '/',
+                target: '/zjgtjy/gpgg',
+            },
+            {
+                title: '拍卖公告',
+                docs: 'https://docs.rsshub.app/government.html',
+                source: '/',
+                target: '/zjgtjy/pmgg',
+            },
+            {
+                title: '补充公告',
+                docs: 'https://docs.rsshub.app/government.html',
+                source: '/',
+                target: '/zjgtjy/bcgg',
+            },
+        ],
+    },
+};

package/lib/v2/zjgtjy/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:type?', require('./index'));
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rsshub",
-  "version": "1.0.0-master.fc91df6",
+  "version": "1.0.0-master.fd40700",
   "description": "Make RSS Great Again!",
   "main": "lib/pkg.js",
   "files": [
@@ -16,9 +16,9 @@
     "build:radar": "node scripts/workflow/build-radar.js",
     "build:maintainer": "node scripts/workflow/build-maintainer.js",
     "lint": "eslint .",
-    "format": "eslint \"**/*.js\" --fix && node docs/.format/format.js && prettier \"**/*.{js,json}\" --write",
-    "format:staged": "eslint \"**/*.js\" --fix && node docs/.format/format.js --staged && pretty-quick --staged --verbose --pattern \"**/*.{js,json}\"",
-    "format:check": "eslint \"**/*.js\" && prettier-check \"**/*.{js,json}\"",
+    "format": "eslint \"**/*.{js,yml}\" --fix && node docs/.format/format.js && prettier \"**/*.{js,json}\" --write",
+    "format:staged": "eslint \"**/*.{js,yml}\" --fix && node docs/.format/format.js --staged && pretty-quick --staged --verbose --pattern \"**/*.{js,json}\"",
+    "format:check": "eslint \"**/*.{js,yml}\" && prettier-check \"**/*.{js,json}\"",
     "test": "npm run format:check && cross-env NODE_ENV=test jest --coverage --runInBand --forceExit",
     "jest": "cross-env NODE_ENV=test jest --runInBand --forceExit",
     "jest:watch": "cross-env NODE_ENV=test jest --watch"
@@ -40,96 +40,96 @@
   },
   "homepage": "https://github.com/DIYgod/RSSHub#readme",
   "devDependencies": {
-    "@napi-rs/pinyin": "^1.6.1",
-    "@types/cheerio": "0.22.24",
-    "@types/got": "9.6.11",
-    "@types/koa": "2.13.1",
-    "@vercel/nft": "0.9.6",
+    "@napi-rs/pinyin": "1.7.0",
+    "@types/cheerio": "0.22.31",
+    "@types/got": "9.6.12",
+    "@types/koa": "2.13.4",
+    "@vercel/nft": "0.17.4",
     "@vuepress/plugin-back-to-top": "1.8.2",
     "@vuepress/plugin-google-analytics": "1.8.2",
     "@vuepress/plugin-pwa": "1.8.2",
-    "ci-info": "^2.0.0",
+    "ci-info": "3.3.0",
     "cross-env": "7.0.3",
-    "eslint": "7.29.0",
+    "eslint": "8.8.0",
     "eslint-config-prettier": "8.3.0",
-    "eslint-plugin-prettier": "3.4.0",
-    "fs-extra": "^8.1.0",
-    "jest": "26.6.3",
+    "eslint-plugin-prettier": "4.0.0",
+    "eslint-plugin-yml": "0.13.0",
+    "fs-extra": "10.0.0",
+    "jest": "27.5.1",
     "mockdate": "3.0.5",
-    "nock": "13.0.11",
-    "nodemon": "2.0.7",
-    "prettier": "2.2.1",
+    "nock": "13.2.4",
+    "nodemon": "2.0.15",
+    "prettier": "2.5.1",
     "prettier-check": "2.0.0",
-    "pretty-quick": "3.1.1",
+    "pretty-quick": "3.1.3",
     "remark": "12.0.1",
     "remark-frontmatter": "2.0.0",
     "remark-pangu": "2.2.0",
     "remark-preset-prettier": "0.4.1",
-    "staged-git-files": "1.2.0",
-    "string-width": "4.2.2",
-    "supertest": "6.1.3",
+    "staged-git-files": "1.3.0",
+    "string-width": "4.2.3",
+    "supertest": "6.2.2",
     "tosource": "2.0.0-alpha.3",
     "vuepress": "1.8.2",
     "yorkie": "2.0.0"
   },
   "dependencies": {
-    "@koa/router": "10.0.0",
-    "@postlight/mercury-parser": "2.2.0",
-    "@sentry/node": "6.7.2",
+    "@koa/router": "10.1.1",
+    "@postlight/mercury-parser": "2.2.1",
+    "@sentry/node": "6.17.6",
     "aes-js": "3.1.2",
     "art-template": "4.13.2",
     "bbcodejs": "0.0.4",
-    "cheerio": "1.0.0-rc.3",
-    "chrono-node": "2.2.6",
-    "city-timezones": "^1.2.0",
-    "crypto-js": "4.0.0",
-    "currency-symbol-map": "4.0.4",
-    "dayjs": "1.10.3",
-    "dotenv": "8.5.1",
+    "cheerio": "1.0.0-rc.10",
+    "chrono-node": "2.3.6",
+    "city-timezones": "1.2.0",
+    "crypto-js": "4.1.1",
+    "currency-symbol-map": "5.0.1",
+    "dayjs": "1.10.7",
+    "dotenv": "16.0.0",
     "emailjs-imap-client": "3.1.0",
-    "entities": "2.1.0",
+    "entities": "3.0.1",
     "etag": "1.8.1",
     "fanfou-sdk": "4.2.0",
-    "fs-extra": "^8.1.0",
-    "git-rev-sync": "3.0.1",
-    "googleapis": "73.0.0",
-    "got": "11.8.2",
+    "git-rev-sync": "3.0.2",
+    "googleapis": "95.0.0",
+    "got": "11.8.3",
     "https-proxy-agent": "5.0.0",
-    "iconv-lite": "0.6.2",
-    "instagram-private-api": "1.44.1",
-    "ioredis": "4.27.2",
+    "iconv-lite": "0.6.3",
+    "instagram-private-api": "1.45.2",
+    "ioredis": "4.28.5",
     "ip": "*",
-    "ip-regex": "4.2.0",
+    "ip-regex": "4.3.0",
     "is-localhost-ip": "1.4.0",
-    "jsdom": "16.5.3",
+    "jsdom": "19.0.0",
     "json-bigint": "1.0.0",
     "json5": "2.2.0",
-    "koa": "2.13.1",
+    "koa": "2.13.4",
     "koa-basic-auth": "4.0.0",
     "koa-favicon": "2.1.0",
     "koa-mount": "4.0.0",
     "lru-cache": "6.0.0",
     "lz-string": "1.4.4",
-    "mailparser": "3.2.0",
-    "markdown-it": "12.0.6",
+    "mailparser": "3.4.0",
+    "markdown-it": "12.2.0",
     "module-alias": "2.2.2",
-    "parse-torrent": "9.1.3",
-    "pidusage": "2.0.21",
-    "plist": "3.0.2",
-    "puppeteer": "9.1.1",
-    "query-string": "7.0.1",
+    "parse-torrent": "9.1.4",
+    "pidusage": "3.0.0",
+    "plist": "3.0.4",
+    "puppeteer": "11.0.0",
+    "query-string": "7.1.1",
     "require-all": "3.0.0",
-    "rss-parser": "3.10.0",
+    "rss-parser": "3.12.0",
     "showdown": "1.9.1",
-    "simplecc-wasm": "0.1.2",
-    "socks-proxy-agent": "6.0.0",
-    "string-similarity": "^4.0.3",
+    "simplecc-wasm": "0.1.4",
+    "socks-proxy-agent": "6.1.1",
+    "string-similarity": "4.0.4",
     "tiny-async-pool": "1.2.0",
-    "torrent-search-api": "^2.1.3",
+    "torrent-search-api": "2.1.3",
     "tough-cookie": "4.0.0",
     "tunnel": "0.0.6",
     "twit": "2.2.11",
-    "winston": "3.3.3"
+    "winston": "3.5.1"
   },
   "jest": {
     "testMatch": [
@@ -152,5 +152,8 @@
     "moduleNameMapper": {
       "@/(.*)": "<rootDir>/lib/$1"
     }
+  },
+  "nodemonConfig": {
+    "ext": "art,js,mjs,json"
   }
 }

package/lib/routes/2048/bbs.js

@@ -1,100 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const fid = ctx.params.fid;
-
-    const base = 'https://hjd2048.com';
-
-    const url = `${base}/2048/thread.php?fid-${fid}-page-1.html`;
-
-    const list_response = await got.get(url);
-    const $ = cheerio.load(list_response.data);
-
-    const list = $('.tr3.t_one').toArray();
-    $('#breadCrumb span.fr').remove();
-    const forum_name = $('#breadCrumb').text().replace(/»/g, '-');
-
-    const parseContent = async (htmlString) => {
-        const $ = cheerio.load(htmlString);
-
-        const time = $('.tiptop.cc > .fl.gray').attr('title');
-        const content = $('.tpc_content');
-
-        const result = {
-            description: content.html(),
-            pubDate: time ? new Date(time) : new Date(),
-        };
-
-        const test_external_torrent = content.find('a').filter(function () {
-            return $(this)
-                .attr('href')
-                .match(/\/list\.php\?name=\w{32}/);
-        });
-
-        if (test_external_torrent.length !== 0) {
-            const torrent_url = test_external_torrent[0].attribs.href;
-            const response = await got.get(torrent_url);
-            const magnet_url = response.data.match(/"(magnet:\?.*?)"/);
-            if (magnet_url) {
-                result.enclosure_url = magnet_url[1];
-                result.enclosure_type = 'application/x-bittorrent';
-            }
-        }
-
-        return result;
-    };
-
-    const out = await Promise.all(
-        list.slice(0, 30).map(async (item) => {
-            const $ = cheerio.load(item);
-
-            if (!$('td > a').first().attr('title')) {
-                return Promise.resolve('');
-            }
-
-            if ($("img[title='置顶帖标志']").length !== 0) {
-                return Promise.resolve('');
-            }
-
-            const title = $('a.subject');
-            const author = $('a.bl');
-            const path = title.attr('href');
-
-            const key = `/2048/${path}`;
-            const link = `${base}/2048/${path}`;
-
-            const cache = await ctx.cache.get(key);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const rssitem = {
-                title: title.text().trim(),
-                author: author.text().trim(),
-                link,
-                guid: key,
-            };
-
-            try {
-                const response = await got.get(link);
-                const result = await parseContent(response.data);
-
-                rssitem.description = result.description;
-                rssitem.pubDate = result.pubDate;
-                rssitem.enclosure_url = result.enclosure_url;
-                rssitem.enclosure_type = result.enclosure_type;
-            } catch (err) {
-                return Promise.resolve('');
-            }
-            ctx.cache.set(key, JSON.stringify(rssitem));
-            return Promise.resolve(rssitem);
-        })
-    );
-
-    ctx.state.data = {
-        title: forum_name,
-        link: url,
-        item: out.filter((item) => item !== ''),
-    };
-};