rsshub 1.0.0-master.fe99193 → 1.0.0-master.fe9c82a

package/lib/v2/csu/utils.js

@@ -0,0 +1,17 @@
+const { inflateSync } = require('zlib');
+
+const unzip = (b64Data) => {
+    const strData = Buffer.from(b64Data, 'base64').toString('binary');
+    const charData = strData.split('').map((x) => x.charCodeAt(0));
+    const binData = new Uint8Array(charData);
+    const data = inflateSync(binData);
+    let ret = '';
+    for (const c of data) {
+        ret += String.fromCharCode(c);
+    }
+    return ret;
+};
+
+module.exports = {
+    unzip,
+};

package/lib/v2/curius/links.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const username = ctx.params.name;
+
+    const name_response = await got(`https://curius.app/api/users/${username}`, {
+        headers: {
+            Referer: `https://curius.app/${username}`,
+        },
+    });
+
+    const data = name_response.data;
+
+    const uid = data.user.id;
+    const name = `${data.user.firstName} ${data.user.lastName}`;
+
+    const response = await got(`https://curius.app/api/users/${uid}/links?page=0`, {
+        headers: {
+            Referer: `https://curius.app/${username}`,
+        },
+    });
+
+    const items = response.data.userSaved.map((item) => ({
+        title: item.title,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        }),
+        link: item.link,
+        pubDate: parseDate(item.createdDate),
+        guid: `curius:${username}:${item.id}`,
+    }));
+
+    ctx.state.data = {
+        title: `${name} - Curius`,
+        link: `https://curius.app/${username}`,
+        description: `${name} - Curius`,
+        allowEmpty: true,
+        item: items,
+    };
+};

package/lib/v2/curius/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/links/:name': ['Ovler-Young'],
+};

package/lib/v2/curius/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'curius.app': {
+        _name: 'Curius',
+        '.': [
+            {
+                title: '用户',
+                docs: 'https://docs.rsshub.app/social-media.html#curius',
+                source: '/:name',
+                target: '/curius/links/:name',
+            },
+        ],
+    },
+};

package/lib/v2/curius/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/links/:name', require('./links'));
+};

package/lib/v2/curius/templates/description.art

@@ -0,0 +1,16 @@
+{{ if item.metadata.full_text }}
+    原文：{{@ item.metadata.full_text.replace(/\n/gm, '<br>') }}<br><br>
+{{ /if }}
+{{ if item.comments.length }}
+    评论：{{ item.comments[0].text.substring(0, 100) }}<br><br>
+{{ /if }}
+{{ if item.highlights.length }}
+    评论：
+    {{ each item.highlights highlight }}
+        {{ if highlight.comment }}
+            {{ highlight.highlight }}<br>评论：{{ highlight.comment.text }}<br>
+        {{ else }}
+            <br>标注：{{ highlight.highlight }}<br>
+        {{ /if }}
+    {{ /each }}
+{{ /if }}

package/lib/v2/cw/author.js

@@ -0,0 +1,18 @@
+const { baseUrl, parsePage } = require('./utils');
+
+module.exports = async (ctx) => {
+    const browser = await require('@/utils/puppeteer')();
+
+    const { $, items } = await parsePage('author', browser, ctx);
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('.authorTxt').text(),
+        link: `${baseUrl}/author/${ctx.params.channel}`,
+        image: $('.authorPhoto img').attr('src') || `${baseUrl}/assets_new/img/fbshare.jpg'`,
+        language: $('meta[property="og:locale"]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/cw/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/author/:channel': ['TonyRL'],
     '/master/:channel': ['TonyRL'],
     '/sub/:channel': ['TonyRL'],
     '/today': ['TonyRL'],

package/lib/v2/cw/master.js

@@ -1,29 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const { channel } = ctx.params;
-    const pageUrl = `${baseUrl}/masterChannel.action`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-        searchParams: {
-            idMasterChannel: channel,
-        },
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('master', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/masterChannel.action?idMasterChannel=${ctx.params.channel}`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/radar.js

@@ -20,6 +20,12 @@ module.exports = {
                 source: ['/subchannel.action'],
                 target: (_, url) => `/cw/sub/${new URL(url).searchParams.get('idSubChannel')}`,
             },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/traditional-media.html#tian-xia-za-zhi',
+                source: ['/author/:channel'],
+                target: '/cw/author/:channel',
+            },
         ],
     },
 };

package/lib/v2/cw/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
+    router.get('/author/:channel', require('./author'));
     router.get('/master/:channel', require('./master'));
     router.get('/sub/:channel', require('./sub'));
     router.get('/today', require('./today'));

package/lib/v2/cw/sub.js

@@ -1,29 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const { channel } = ctx.params;
-    const pageUrl = `${baseUrl}/subchannel.action`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-        searchParams: {
-            idSubChannel: channel,
-        },
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('sub', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/subchannel.action?idSubChannel=${ctx.params.channel}`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/today.js

@@ -1,25 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const pageUrl = `${baseUrl}/today`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('today', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/today`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/utils.js

@@ -1,26 +1,73 @@
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-const { Cookie, CookieJar } = require('tough-cookie');
-let cookieJar;
-const config = require('@/config').value;
+const { getCookies, setCookies } = require('@/utils/puppeteer-utils');
+const logger = require('@/utils/logger');
+let cookie;
 
 const baseUrl = 'https://www.cw.com.tw';
 
-const got = require('@/utils/got').extend({
-    headers: {
-        'User-Agent': config.trueUA,
+const pathMap = {
+    today: {
+        pageUrl: () => '/today',
+        limit: 30,
     },
-});
+    master: {
+        pageUrl: (channel) => `/masterChannel.action?idMasterChannel=${channel}`,
+        limit: 12,
+    },
+    sub: {
+        pageUrl: (channel) => `/subchannel.action?idSubChannel=${channel}`,
+        limit: 12,
+    },
+    author: {
+        pageUrl: (channel) => `/author/${channel}`,
+        limit: 10,
+    },
+};
 
-const getCookie = async () => {
-    const response = await got(`${baseUrl}/user/get/cookie-bar`);
-    const cookies = response.headers['set-cookie'];
-    if (Array.isArray(cookies)) {
-        cookieJar = cookies.map(Cookie.parse);
-    } else {
-        cookieJar = [Cookie.parse(cookieJar)];
+const getCookie = async (browser, tryGet) => {
+    if (!cookie) {
+        cookie = await tryGet('cw:cookie', async () => {
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
+            logger.debug(`Requesting ${baseUrl}/user/get/cookie-bar`);
+            await page.goto(`${baseUrl}/user/get/cookie-bar`, {
+                waitUntil: 'domcontentloaded',
+            });
+            cookie = await getCookies(page);
+            await page.close();
+            return cookie;
+        });
     }
-    cookieJar = CookieJar.fromJSON({ cookies: cookieJar });
+    return cookie;
+};
+
+const parsePage = async (path, browser, ctx) => {
+    const pageUrl = `${baseUrl}${pathMap[path].pageUrl(ctx.params.channel)}`;
+
+    const cookie = await getCookie(browser, ctx.cache.tryGet);
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await setCookies(page, cookie, 'cw.com.tw');
+    logger.debug(`Requesting ${pageUrl}`);
+    await page.goto(pageUrl, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    const response = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    const $ = cheerio.load(response);
+
+    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : pathMap[path].limit);
+    const items = await parseItems(list, browser, ctx.cache.tryGet);
+
+    return { $, items };
 };
 
 const parseList = ($, limit) =>
@@ -36,15 +83,25 @@ const parseList = ($, limit) =>
         })
         .slice(0, limit);
 
-const parseItems = (list, tryGet) =>
+const parseItems = (list, browser, tryGet) =>
     Promise.all(
         list.map((item) =>
             tryGet(item.link, async () => {
-                const { data: response } = await got(item.link, {
-                    cookieJar,
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await setCookies(page, cookie, 'cw.com.tw');
+                logger.debug(`Requesting ${item.link}`);
+                await page.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
                 });
 
+                const response = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
                 const $ = cheerio.load(response);
+
                 const meta = JSON.parse($('head script[type="application/ld+json"]').eq(0).text());
                 $('.article__head .breadcrumb, .article__head h1, .article__provideViews, .ad').remove();
                 $('img.lazyload').each((_, img) => {
@@ -67,9 +124,10 @@ const parseItems = (list, tryGet) =>
 
 module.exports = {
     baseUrl,
-    cookieJar,
-    got,
+    pathMap,
     getCookie,
+    setCookies,
+    parsePage,
     parseList,
     parseItems,
 };

package/lib/v2/dcard/section.js

@@ -38,7 +38,7 @@ module.exports = async (ctx) => {
     await page.goto(`${api}&limit=100`);
     await page.waitForSelector('body > pre');
     const response = await page.evaluate(() => document.querySelector('body > pre').innerText);
-    const cookies = await ctx.cache.tryGet('dcard:cookies', async () => await page.cookies(), 3600, false);
+    const cookies = await ctx.cache.tryGet('dcard:cookies', () => page.cookies(), 3600, false);
     await page.close();
 
     const data = JSON.parse(response);

package/lib/v2/dcfever/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/news/:type?': ['TonyRL'],
+    '/reviews/:type?': ['TonyRL'],
+    '/trading/search/:keyword/:mainCat?': ['TonyRL'],
+    '/trading/:id': ['TonyRL'],
+};

package/lib/v2/dcfever/news.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { baseUrl, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { type } = ctx.params;
+
+    const response = await got(`${baseUrl}/news/index.php`, {
+        searchParams: {
+            type: type ? type : undefined,
+        },
+    });
+    const $ = cheerio.load(response.data);
+
+    const list = $('.col-md-left .title a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: new URL(item.attr('href'), response.url).href,
+            };
+        });
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `${$('.channel_nav')
+            .contents()
+            .filter((_, e) => e.nodeType === 3)
+            .text()} - ${$('head title').text()}`,
+        link: response.url,
+        image: 'https://cdn10.dcfever.com/images/android_192.png',
+        item: items,
+    };
+};

package/lib/v2/dcfever/radar.js

@@ -0,0 +1,37 @@
+module.exports = {
+    'dcfever.com': {
+        _name: 'DCFever',
+        '.': [
+            {
+                title: '新聞中心',
+                docs: 'https://docs.rsshub.app/new-media.html#dcfever',
+                source: ['/news/index.php', '/'],
+                target: (_, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    return `/dcfever/news${searchParams.has('type') ? `/${new URL(url).searchParams.get('type')}` : ''}`;
+                },
+            },
+            {
+                title: '測試報告',
+                docs: 'https://docs.rsshub.app/new-media.html#dcfever',
+                source: ['/:type/reviews.php'],
+                target: '/dcfever/reviews/:type',
+            },
+            {
+                title: '二手市集',
+                docs: 'https://docs.rsshub.app/new-media.html#dcfever',
+                source: ['/trading/listing.php'],
+                target: (_, url) => `/dcfever/trading/${new URL(url).searchParams.get('id')}`,
+            },
+            {
+                title: '二手市集 - 物品搜尋',
+                docs: 'https://docs.rsshub.app/new-media.html#dcfever',
+                source: ['/trading/search.php'],
+                target: (_, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    return `/dcfever/trading/search/${searchParams.get('keyword')}${searchParams.has('main_cat') ? `/${searchParams.get('main_cat')}` : ''}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/dcfever/reviews.js

@@ -0,0 +1,29 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { baseUrl, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { type = 'cameras' } = ctx.params;
+
+    const response = await got(`${baseUrl}/${type}/reviews.php`);
+    const $ = cheerio.load(response.data);
+
+    const list = $('.col-md-left .title a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: new URL(item.attr('href'), response.url).href,
+            };
+        });
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: response.url,
+        image: 'https://cdn10.dcfever.com/images/android_192.png',
+        item: items,
+    };
+};

package/lib/v2/dcfever/router.js

@@ -0,0 +1,6 @@
+module.exports = (router) => {
+    router.get('/news/:type?', require('./news'));
+    router.get('/reviews/:type?', require('./reviews'));
+    router.get('/trading/search/:keyword/:mainCat?', require('./trading-search'));
+    router.get('/trading/:id', require('./trading'));
+};

package/lib/v2/dcfever/templates/trading.art

@@ -0,0 +1,14 @@
+<h2>{{ info.find('.trading_item_title').text() }}</h2>
+{{ info.find('.trading_item_type_tag').text() }} {{ info.find('.trading_item_price').text() }}<br>
+<table>
+    <tr><th>賣家</th><td>{{ info.find('.clearfix .content').eq(0).text() }}</td></tr>
+    <tr><th>查詢次數</th><td>{{ info.find('.clearfix .content').eq(1).text() }}</td></tr>
+    <tr><th>瀏覽次數</th><td>{{ info.find('.clearfix .content').eq(2).text() }}</td></tr>
+    <tr><th>刊登日期</th><td>{{ info.find('.clearfix .content').eq(3).text() }}</td></tr>
+    <tr><th>最後更新</th><td>{{ info.find('.clearfix .content').eq(4).text() }}</td></tr>
+    <tr><th>刊登期至</th><td>{{ info.find('.clearfix .content').eq(5).text() }}</td></tr>
+    <tr><th>刊登狀態</th><td>{{ info.find('.clearfix .content').eq(6).text() }}</td></tr>
+</table><br>
+
+{{@ description }}<br><br>
+{{@ photo }}

package/lib/v2/dcfever/trading-search.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+// const { parseRelativeDate } = require('@/utils/parse-date');
+const { baseUrl, parseTradeItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { keyword, mainCat } = ctx.params;
+
+    const response = await got(`${baseUrl}/trading/search.php`, {
+        searchParams: {
+            keyword,
+            type: 'all',
+            main_cat: mainCat,
+            form_action: 'search_action',
+        },
+    });
+    const $ = cheerio.load(response.data);
+
+    const list = $('.item_list li a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            item.find('.optional').remove();
+            return {
+                title: item.find('.trade_title').text(),
+                link: new URL(item.attr('href'), response.url).href,
+                author: item.find('.trade_info').text(),
+            };
+        });
+
+    const items = await Promise.all(list.map((item) => parseTradeItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: response.url,
+        image: 'https://cdn10.dcfever.com/images/android_192.png',
+        item: items,
+    };
+};

package/lib/v2/dcfever/trading.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+// const { parseRelativeDate } = require('@/utils/parse-date');
+const { baseUrl, parseTradeItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { id, order = 'new' } = ctx.params;
+
+    const response = await got(`${baseUrl}/trading/listing.php`, {
+        searchParams: {
+            id,
+            order,
+            type: 'all',
+        },
+    });
+    const $ = cheerio.load(response.data);
+
+    const list = $('.item_list li a')
+        .toArray()
+        .filter((item) => $(item).attr('href') !== '/documents/advertising.php')
+        .map((item) => {
+            item = $(item);
+            item.find('.optional').remove();
+            return {
+                title: item.find('.trade_title').text(),
+                link: new URL(item.attr('href'), response.url).href,
+                author: item.find('.trade_info').text(),
+            };
+        });
+
+    const items = await Promise.all(list.map((item) => parseTradeItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: response.url,
+        image: 'https://cdn10.dcfever.com/images/android_192.png',
+        item: items,
+    };
+};