rsshub 1.0.0-master.ff487f3 → 1.0.0-master.ff76aa7

package/lib/routes/parliament.uk/lordslibrary.ts

@@ -0,0 +1,55 @@
+import { load } from 'cheerio';
+import { Route } from '@/types';
+import puppeteer from '@/utils/puppeteer';
+import timezone from '@/utils/timezone';
+
+export const route: Route = {
+    path: '/lordslibrary/type/:topic?',
+    categories: ['government'],
+    example: '/parliament.uk/lordslibrary/type/research-briefing',
+    parameters: { topic: 'research by topic, string, example: [research-briefing|buisness|economy]' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: true,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'House of Lords Library',
+    maintainers: ['AntiKnot'],
+    handler,
+};
+
+async function handler(ctx) {
+    const { topic } = ctx.req.param();
+    const baseUrl = 'https://lordslibrary.parliament.uk';
+    const url = `${baseUrl}/type/${topic}/`;
+    const browser = await puppeteer();
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    const $ = load(html);
+    const items = $('div.l-box.l-box--no-border.card__text')
+        .map((_, article) => ({
+            title: $(article).find('.card__text a').text().trim(),
+            link: $(article).find('.card__text a').attr('href'),
+            description: $(article).find('p').last().text().trim(),
+            pubDate: timezone($(article).find('.card__date time').attr('datetime')),
+        }))
+        .toArray();
+    browser.close();
+    return {
+        title: `parliament - lordslibrary - ${topic}`,
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/parliament.uk/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'UK Parliament',
+    url: 'parliament.uk',
+};

package/lib/routes/pconline/focus.ts

@@ -0,0 +1,143 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+const rootUrl = 'https://www.pconline.com.cn';
+interface Item {
+    id: string;
+    title: string;
+    channelName: string;
+    wap_3g_url: string;
+    bigImage: string;
+    cover: string;
+    largeImage: string;
+    authorname: string;
+    authorImg: string;
+    pc_pubDate: string;
+    artType: string;
+    summary: string;
+    url: string;
+}
+
+const categories = {
+    all: {
+        title: '全部',
+        path: '',
+    },
+    tech: {
+        title: '科技',
+        path: 'tech/',
+    },
+    finance: {
+        title: '财经',
+        path: 'finance/',
+    },
+    life: {
+        title: '生活',
+        path: 'life/',
+    },
+    company: {
+        title: '公司',
+        path: 'company/',
+    },
+    character: {
+        title: '人物',
+        path: 'character/',
+    },
+};
+
+const getContent = (item) =>
+    cache.tryGet(item.link, async () => {
+        const detailResponse = await got({
+            method: 'get',
+            url: `https:${item.link}`,
+            responseType: 'arrayBuffer',
+        });
+
+        const utf8decoder = new TextDecoder('GBK');
+        const html = utf8decoder.decode(detailResponse.data);
+        const $ = load(html);
+        item.description = $('.context-box .context-table tbody td').html();
+        return item;
+    });
+
+export const handler = async (ctx) => {
+    const { category = 'all' } = ctx.req.param();
+    const cate = categories[category] || categories.all;
+    const currentUrl = `${rootUrl}/3g/other/focus/${cate.path}index.html`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const resString = response.data
+        .replace(/Module\.callback\((.*)\)/s, '$1')
+        .split('\n')
+        .filter((e) => e.indexOf('"tags":') !== 0)
+        .join('\n')
+        .replaceAll("'", '"');
+    const tinyData = resString.replaceAll(/[\n\r]/g, '');
+    const dataString = tinyData.replaceAll(',}', '}');
+    const data = JSON.parse(dataString || '');
+    const { articleList } = data;
+    const list = articleList.map((item: Item) => ({
+        id: item.id,
+        title: item.title,
+        author: [
+            {
+                name: item.authorname,
+                avatar: item.authorImg,
+            },
+        ],
+        pubDate: parseDate(item.pc_pubDate),
+        link: item.url,
+        description: item.summary,
+        category: item.channelName,
+        image: item.cover,
+    }));
+
+    const items = await Promise.all(list.map((item) => getContent(item)));
+
+    return {
+        title: `太平洋科技-${cate.title}`,
+        link: currentUrl,
+        item: items,
+    };
+};
+
+export const route: Route = {
+    path: '/focus/:category?',
+    categories: ['new-media'],
+    example: '/pconline/focus',
+    parameters: {
+        category: {
+            description: '科技新闻的类别，获取最新的一页，分别：all, tech, finance, life, company, character',
+            default: 'all',
+        },
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['pconline.com.cn/focus/', 'pconline.com.cn/'],
+            target: '/focus',
+        },
+    ],
+    name: '科技新闻',
+    maintainers: ['CH563'],
+    handler,
+    description: `::: tip
+| 全部 | 科技 | 财经 | 生活 | 公司 | 人物 |
+| --- | --- | --- | --- | --- | --- |
+| all | tech | finance | life | company | character |
+:::`,
+};

package/lib/routes/pconline/namespace.ts

@@ -0,0 +1,10 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '太平洋科技',
+    url: 'pconline.com.cn',
+    description: `
+:::tip
+太平洋科技是专业IT门户网站,为用户和经销商提供IT资讯和行情报价,涉及电脑,手机,数码产品,软件等.
+:::`,
+};

package/lib/routes/people/index.ts

@@ -32,8 +32,12 @@ async function handler(ctx) {
         responseType: 'buffer',
     });
 
-    const encoding = site === 'www' ? 'gbk' : 'utf-8';
-    const decodedResponse = iconv.decode(response, encoding);
+    // not seen Content-Type in response headers
+    // try to parse charset from meta tag
+    let decodedResponse = iconv.decode(response, 'utf-8');
+    const parsedCharset = decodedResponse.match(/<meta.*?charset=["']?([^"'>]+)["']?/i);
+    const encoding = parsedCharset ? parsedCharset[1].toLowerCase() : 'utf-8';
+    decodedResponse = encoding === 'utf-8' ? decodedResponse : iconv.decode(response, encoding);
     const $ = load(decodedResponse);
 
     $('em').remove();

package/lib/routes/picnob/user.ts

@@ -82,7 +82,8 @@ async function handler(ctx) {
 
     const list = await Promise.all(
         posts.items.map(async (item) => {
-            const { shortcode, type, sum_pure, time } = item;
+            const { shortcode, sum, type, time } = item;
+
             const link = `${baseUrl}/post/${shortcode}/`;
             if (type === 'img_multi') {
                 item.images = await cache.tryGet(link, async () => {
@@ -111,8 +112,15 @@ async function handler(ctx) {
             }
 
             return {
-                title: sum_pure,
-                description: art(path.join(__dirname, 'templates/desc.art'), { item }),
+                // sum_pure lacks linebreaks/spaces between lines
+                title: load(sum, null, false).text().replaceAll('\n', ' '),
+                description: art(path.join(__dirname, 'templates/desc.art'), {
+                    item: {
+                        ...item,
+                        // Fix linebreaks
+                        sum: sum.replaceAll('\n', '<br>'),
+                    },
+                }),
                 link,
                 pubDate: parseDate(time, 'X'),
             };

package/lib/routes/picuki/profile.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { DataItem, Route } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
@@ -16,20 +16,29 @@ function deVideo(media) {
     let media_deVideo = '';
     $('img,video').each((_, medium) => {
         const tag = medium.name;
-        medium = $(medium);
-        const poster = medium.attr('poster');
+        const mediumElement = $(medium);
+        const poster = mediumElement.attr('poster');
         // 如果有 poster 属性，表明它是视频，将它替换为它的 poster；如果不是，就原样返回
-        media_deVideo += poster ? `<img src='${poster}' alt='video poster'>` : tag === 'img' ? medium.toString() : '';
+        media_deVideo += (() => {
+            if (poster) {
+                return `<img src='${poster}' alt='video poster'>`;
+            } else if (tag === 'img') {
+                return mediumElement.toString();
+            } else {
+                return '';
+            }
+        })();
     });
     return media_deVideo;
 }
 
 export const route: Route = {
-    path: '/profile/:id/:functionalFlag?',
+    path: '/profile/:id/:type?/:functionalFlag?',
     categories: ['social-media'],
     example: '/picuki/profile/stefaniejoosten',
     parameters: {
         id: 'Instagram user id',
+        type: 'Type of profile page (profile or tagged)',
         functionalFlag: `functional flag, see the table below
 | functionalFlag | Video embedding                         | Fetching Instagram Stories |
 | -------------- | --------------------------------------- | -------------------------- |
@@ -51,9 +60,13 @@ export const route: Route = {
             source: ['www.picuki.com/profile/:id'],
             target: '/profile/:id',
         },
+        {
+            source: ['www.picuki.com/profile-tagged/:id'],
+            target: '/profile/:id/tagged',
+        },
     ],
     name: 'User Profile - Picuki',
-    maintainers: ['hoilc', 'Rongronggg9', 'devinmugen'],
+    maintainers: ['hoilc', 'Rongronggg9', 'devinmugen', 'NekoAria'],
     handler,
     description: `
   :::warning
@@ -67,61 +80,58 @@ async function handler(ctx) {
     const browser = await puppeteer();
 
     const id = ctx.req.param('id');
-    const displayVideo = ctx.req.param('functionalFlag') !== '0';
-    const includeStories = ctx.req.param('functionalFlag') === '10';
+    const type = ctx.req.param('type') ?? 'profile';
+    const functionalFlag = ctx.req.param('functionalFlag') ?? '1';
+    const displayVideo = functionalFlag !== '0';
+    const includeStories = functionalFlag === '10';
 
-    const profileUrl = `https://www.picuki.com/profile/${id}`;
+    const profileUrl = `https://www.picuki.com/${type === 'tagged' ? 'profile-tagged' : 'profile'}/${id}`;
 
-    const data = await cache.tryGet(
-        `picuki-${id}-profile-${includeStories}`,
-        async () => {
-            const _r = await puppeteerGet(profileUrl, browser, includeStories);
-            return _r;
-        },
-        config.cache.routeExpire,
-        false
-    );
+    const key = `picuki-${id}-${type}-${includeStories}`;
+    const data = await cache.tryGet(key, () => puppeteerGet(profileUrl, browser, includeStories), config.cache.routeExpire, false);
     const $ = load(data);
 
     const profileName = $('.profile-name-bottom').text();
+    const profileTitle = type === 'tagged' ? `${profileName} (@${id}) tagged posts - Picuki` : `${profileName} (@${id}) public posts - Picuki`;
     const profileImg = $('.profile-avatar > img').attr('src');
     const profileDescription = $('.profile-description').text();
 
-    const list = $('ul.box-photos [data-s="media"]').get();
+    const list = $('ul.box-photos [data-s="media"]').toArray();
+
+    let items: DataItem[] = [];
 
-    let items = [];
+    let description: string;
 
     if (includeStories) {
         const stories_wrapper = $('.stories_wrapper');
         if (stories_wrapper.length) {
             const storyItems = $(stories_wrapper)
                 .find('.item')
-                .get()
+                .toArray()
                 .map((item) => {
                     const $item = $(item);
-                    let title = $item.find('.stories_count');
-                    title = title.length ? title.text() : '';
-                    const pubDate = title ? chrono.parseDate(title) : new Date();
+                    const titleElement = $item.find('.stories_count');
+                    const title = titleElement.length ? titleElement.text() : '';
+                    const pubDate = chrono.parseDate(title) || new Date();
                     const postBox = $item.find('.launchLightbox');
                     const poster = postBox.attr('data-video-poster');
                     const href = postBox.attr('href');
                     const type = postBox.attr('data-post-type'); // video / image
                     const origin = postBox.attr('data-origin');
                     const storiesBackground = $item.find('.stories_background');
-                    const storiesBackgroundUrl = storiesBackground && storiesBackground.css('background-image').match(/url\('?(.*)'?\)/);
-                    const storiesBackgroundUrlSrc = storiesBackgroundUrl && storiesBackgroundUrl[1];
-                    let description;
+                    const storiesBackgroundUrl = storiesBackground?.css('background-image')?.match(/url\('?(.*)'?\)/);
+                    const storiesBackgroundUrlSrc = storiesBackgroundUrl?.[1];
                     if (type === 'video') {
                         description = art(path.join(__dirname, 'templates/video.art'), {
                             videoSrcs: [href, origin].filter(Boolean),
-                            videoPoster: poster || storiesBackgroundUrlSrc || '',
+                            videoPoster: poster ?? storiesBackgroundUrlSrc ?? '',
                         });
                     } else if (type === 'image') {
-                        description = `<img src="${href || poster || storiesBackgroundUrlSrc || ''}" alt="Instagram Story">`;
+                        description = `<img src="${href ?? poster ?? storiesBackgroundUrlSrc ?? ''}" alt="Instagram Story">`;
                     }
 
                     return {
-                        title: 'Instagram Story' + (pubDate ? '' : `: ${title}`),
+                        title: title ? `Instagram Story: ${title}` : 'Instagram Story',
                         author: `@${id}`,
                         description,
                         link: href,
@@ -149,8 +159,8 @@ async function handler(ctx) {
         $('.single-photo img').each((_, item) => {
             html += $(item).toString();
         });
-        $('.single-photo video').each((_, item) => {
-            item = $(item);
+        $('.single-photo video').each((_, element) => {
+            const item = $(element);
             let videoSrc = item.attr('src');
             if (videoSrc === undefined) {
                 videoSrc = item.children().attr('src');
@@ -158,8 +168,9 @@ async function handler(ctx) {
             const videoPoster = item.attr('poster');
             let origin = item.parent().attr('onclick');
             if (origin) {
-                origin = origin.match(/window\.open\('([^']*)'/);
-                origin = origin && origin[1];
+                const regex = /window\.open\('([^']*)'/;
+                const match = regex.exec(origin);
+                origin = match?.[1];
             }
             html += art(path.join(__dirname, 'templates/video.art'), {
                 videoSrcs: [videoSrc, origin].filter(Boolean),
@@ -172,15 +183,15 @@ async function handler(ctx) {
     items = [
         ...items,
         ...(await Promise.all(
-            list.map(async (post) => {
-                post = $(post);
+            list.map(async (item) => {
+                const post = $(item);
 
                 const postLink = post.find('.photo > a').attr('href');
                 const postTime = post.find('.time');
-                const pubDate = postTime ? chrono.parseDate(postTime.text()) : new Date();
+                const pubDate = postTime?.text() ? (chrono.parseDate(postTime.text()) ?? new Date()) : new Date();
                 const media_displayVideo = await getMedia(postLink);
                 const postText = post
-                    .find('.photo-description')
+                    .find('.photo-action-description')
                     .text()
                     .trim()
                     .replaceAll(/[^\S\n]+/g, ' ')
@@ -206,7 +217,7 @@ async function handler(ctx) {
     await browser.close();
 
     return {
-        title: `${profileName} (@${id}) - Picuki`,
+        title: profileTitle,
         link: profileUrl,
         image: profileImg,
         description: profileDescription,

package/lib/routes/pixiv/ranking.ts

@@ -124,6 +124,7 @@ export const route: Route = {
                     label: 'R-18G rank',
                 },
             ],
+            default: 'day',
         },
         date: 'format: `2018-4-25`',
     },

package/lib/routes/pixivision/index.ts

@@ -0,0 +1,84 @@
+import { Route, DataItem, Data } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import { processContent } from './utils';
+
+export const route: Route = {
+    path: '/:lang/:category?',
+    categories: ['anime'],
+    example: '/pixivision/zh-tw',
+    parameters: { lang: 'Language', category: 'Category' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Category',
+    maintainers: ['SnowAgar25'],
+    description: `:::tip
+  \`https://www.pixivision.net/zh-tw/c/interview\` → \`/pixivision/zh-tw/interview\`
+  :::`,
+    radar: [
+        {
+            source: ['www.pixivision.net/:lang'],
+            target: '/:lang',
+        },
+        {
+            source: ['www.pixivision.net/:lang/c/:category'],
+            target: '/:lang/:category',
+        },
+    ],
+    handler,
+};
+
+async function handler(ctx): Promise<Data> {
+    const { lang, category } = ctx.req.param();
+    const baseUrl = 'https://www.pixivision.net';
+    const url = category ? `${baseUrl}/${lang}/c/${category}` : `${baseUrl}/${lang}`;
+
+    const headers = {
+        headers: {
+            Cookie: `user_lang=${lang.replace('-', '_')}`, // zh-tw → zh_tw
+        },
+    };
+
+    const { data: response } = await got(url, headers);
+    const $ = load(response);
+
+    const list = $('li.article-card-container a[data-gtm-action="ClickTitle"]')
+        .map((_, elem) => ({
+            title: $(elem).text(),
+            link: new URL($(elem).attr('href') ?? '', baseUrl).href,
+        }))
+        .toArray();
+
+    const items = await Promise.all(
+        list.map(async (item) => {
+            const result = await cache.tryGet(item.link, async () => {
+                const { data: articleData } = await got(item.link, headers);
+                const $article = load(articleData);
+
+                const processedDescription = processContent($article, lang);
+
+                return {
+                    title: item.title,
+                    description: processedDescription,
+                    link: item.link,
+                    pubDate: parseDate($article('time').attr('datetime') ?? ''),
+                } as DataItem;
+            });
+            return result;
+        })
+    );
+
+    return {
+        title: `${$('.ssc__header').length ? $('.ssc__header').text() : 'New'} - pixivision`,
+        link: url,
+        item: items.filter((item): item is DataItem => !!item),
+    };
+}

package/lib/routes/pixivision/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'pixivision',
+    url: 'www.pixivision.net',
+};

package/lib/routes/pixivision/utils.ts

@@ -0,0 +1,86 @@
+import { CheerioAPI } from 'cheerio';
+import { config } from '@/config';
+
+const multiImagePrompt = {
+    en: (count) => `${count} images in total`,
+    zh: (count) => `共${count}张图`,
+    'zh-tw': (count) => `共${count}張圖`,
+    ko: (count) => `총 ${count}개의 이미지`,
+    ja: (count) => `計${count}枚の画像`,
+};
+
+export function processContent($: CheerioAPI, lang: string): string {
+    // 移除作者頭像
+    $('.am__work__user-icon-container').remove();
+
+    // 插畫標題&作者
+    $('.am__work__title').attr('style', 'display: inline;');
+    $('.am__work__user-name').attr('style', 'display: inline; margin-left: 10px;');
+
+    // 處理多張圖片的提示
+    $('.mic__label').each((_, elem) => {
+        const $label = $(elem);
+        const count = $label.text();
+        const $workContainer = $label.parentsUntil('.am__work').last().parent();
+        const $titleContainer = $workContainer.find('.am__work__title-container');
+
+        $titleContainer.append(`<p style="float: right; margin: 0;">${multiImagePrompt[lang](count)}</p>`);
+        $label.remove();
+    });
+
+    // 插畫間隔
+    $('.article-item, ._feature-article-body__pixiv_illust').after('<br>');
+
+    // Remove Label & Tags
+    $('.arc__thumbnail-label').remove();
+    $('.arc__footer-container').remove();
+
+    // pixivision card
+    $('article._article-card').each((_, article) => {
+        const $article = $(article);
+
+        const $thumbnail = $article.find('._thumbnail');
+        const thumbnailStyle = $thumbnail.attr('style');
+        const bgImageMatch = thumbnailStyle?.match(/url\((.*?)\)/);
+        const imageUrl = bgImageMatch ? bgImageMatch[1] : '';
+
+        $thumbnail.remove();
+
+        if (imageUrl) {
+            $article.prepend(`<img src="${imageUrl}" alt="Article thumbnail">`);
+        }
+    });
+
+    // 處理 tweet
+    $('.fab__script').each((_, elem) => {
+        const $elem = $(elem);
+        const $link = $elem.find('blockquote > a');
+        const href = $link.attr('href');
+
+        if (href) {
+            const match = href.match(/\/status\/(\d+)/);
+            if (match) {
+                const tweetId = match[1];
+                $elem.html(`
+                <iframe
+                    scrolling="no"
+                    frameborder="0"
+                    allowtransparency="true"
+                    allowfullscreen="true"
+                    class=""
+                    style="position: static; visibility: visible; display: block; width: 550px; height: 1000px; flex-grow: 1;"
+                    title="X Post"
+                    src="https://platform.twitter.com/embed/Tweet.html?id=${tweetId}"
+                ></iframe>
+            `);
+                $elem.find('blockquote').remove();
+            }
+        }
+    });
+
+    return (
+        $('.am__body')
+            .html()
+            ?.replace(/https:\/\/i\.pximg\.net/g, config.pixiv.imgProxy || '') || ''
+    );
+}

package/lib/routes/pku/bbs/hot.ts

@@ -44,7 +44,7 @@ async function handler() {
     const listItems = $('#list-content .list-item')
         .map(function () {
             return {
-                url: new URL($(this).find('> a.link').attr('href'), r.url).href,
+                url: new URL($(this).find('> a.link').attr('href'), 'https://bbs.pku.edu.cn/v2/').href,
                 title: $(this).find('.title').text(),
             };
         })

package/lib/routes/pornhub/category.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
@@ -7,7 +7,8 @@ import { config } from '@/config';
 
 export const route: Route = {
     path: '/category/:caty',
-    categories: ['multimedia'],
+    categories: ['multimedia', 'popular'],
+    view: ViewType.Videos,
     example: '/pornhub/category/popular-with-women',
     parameters: { caty: 'category, see [categories](https://www.pornhub.com/webmasters/categories)' },
     features: {