rsshub 1.0.0-master.ff487f3 → 1.0.0-master.ff76aa7

package/lib/routes/yilinzazhi/latest.ts

@@ -0,0 +1,103 @@
+import { Data, DataItem, Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import dayjs from 'dayjs';
+
+export const route: Route = {
+    path: '/latest',
+    categories: ['reading'],
+    example: '/yilinzazhi/latest',
+    radar: [
+        {
+            source: ['www.yilinzazhi.com'],
+            target: '/',
+        },
+    ],
+    name: '近期文章汇总',
+    maintainers: ['g0ngjie'],
+    handler,
+    url: 'www.yilinzazhi.com',
+    description: '最近一期的文章汇总',
+};
+
+type Stage = {
+    link: string;
+    title: string;
+};
+
+type Catalog = {
+    title: string;
+    tables: Data[];
+};
+
+async function handler(): Promise<Data> {
+    const baseUrl = 'https://www.yilinzazhi.com/';
+    const response = await got(baseUrl);
+    const $ = load(response.data);
+
+    const currentYear = dayjs().year();
+    const yearSection = $('.year-section')
+        .toArray()
+        .find((el) =>
+            $(el)
+                .find('.year-title')
+                .text()
+                .includes(currentYear + '')
+        );
+
+    const stage = $(yearSection!)
+        .find('a')
+        .map<typeof yearSection, Stage>(function () {
+            const aTag = $(this);
+            const link = baseUrl + aTag.attr('href');
+            const title = aTag.text();
+            return { link, title };
+        })
+        .toArray()[0];
+
+    const catalogs = (await cache.tryGet(stage.link, async () => {
+        const stageRes = await got(stage.link);
+        const $$ = load(stageRes.data);
+        const catalogsEl = $$('.maglistbox dl').toArray();
+        const children = catalogsEl.map<Catalog>((catalog) => {
+            const title = $$(catalog).find('dt span').text();
+            const tables = $$(catalog)
+                .find('a')
+                .toArray()
+                .map<Data>((aTag) => {
+                    const href = $$(aTag).attr('href')!;
+                    const yearType = currentYear + href.substring(4, 5);
+                    return {
+                        title: $$(aTag).text(),
+                        link: `${baseUrl}${currentYear}/yl${yearType}/${href}`,
+                    };
+                });
+            return { title, tables };
+        });
+        return children;
+    })) as Catalog[];
+
+    const contents: Data[] = catalogs.flatMap((catalog) => catalog.tables);
+
+    const items = (await Promise.all(
+        contents.map(
+            async (target) =>
+                await cache.tryGet(target.link!, async () => {
+                    const detailRes = await got(target.link);
+                    const $$ = load(detailRes.data);
+                    const detailContainer = $$('.blkContainerSblk.collectionContainer');
+
+                    target.description = detailContainer.html()!;
+
+                    return target;
+                })
+        )
+    )) as DataItem[];
+
+    return {
+        title: '意林 - 近期文章汇总',
+        link: stage.link,
+        item: items,
+    };
+}

package/lib/routes/yilinzazhi/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '意林杂志',
+    url: 'www.yilinzazhi.com',
+    categories: ['reading'],
+    description: '',
+};

package/lib/routes/youtube/playlist.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import utils from './utils';
 import { config } from '@/config';
@@ -7,7 +7,8 @@ import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 export const route: Route = {
     path: '/playlist/:id/:embed?',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
+    view: ViewType.Videos,
     example: '/youtube/playlist/PLqQ1RwlxOgeLTJ1f3fNMSwhjVgaWKo_9Z',
     parameters: { id: 'YouTube playlist id', embed: 'Default to embed the video, set to any value to disable embedding' },
     features: {

package/lib/routes/yystv/docs.ts

@@ -1,7 +1,8 @@
-import { Route } from '@/types';
-import got from '@/utils/got';
+import type { Route, DataItem } from '@/types';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import { parseRelativeDate } from '@/utils/parse-date';
+import cache from '@/utils/cache';
 
 export const route: Route = {
     path: '/docs',
@@ -22,34 +23,42 @@ export const route: Route = {
         },
     ],
     name: '游研社 - 全部文章',
-    maintainers: ['HaitianLiu'],
+    maintainers: ['HaitianLiu', 'yy4382'],
     handler,
     url: 'yystv.cn/docs',
 };
 
 async function handler() {
     const url = `https://www.yystv.cn/docs`;
-    const response = await got({
-        method: 'get',
-        url,
-    });
+    const response = await ofetch(url);
 
-    const data = response.data;
-    const $ = load(data);
+    const $ = load(response);
 
-    const items = $('.list-container li')
-        .slice(0, 18)
-        .map(function () {
+    const itemList = $('.list-container li')
+        .toArray()
+        .map((item) => {
+            const itemElement = $(item);
             const info = {
-                title: $('.list-article-title', this).text(),
-                link: 'https://www.yystv.cn' + $('a', this).attr('href'),
-                pubDate: parseRelativeDate($('.c-999', this).text()),
-                author: $('.handler-author-link', this).text(),
-                description: $('.list-article-intro', this).text(),
+                title: itemElement.find('.list-article-title').text(),
+                link: 'https://www.yystv.cn' + itemElement.find('a').attr('href'),
+                pubDate: parseRelativeDate(itemElement.find('.c-999').text()),
+                author: itemElement.find('.handler-author-link').text(),
+                description: itemElement.find('.list-article-intro').text(),
             };
             return info;
-        })
-        .get();
+        }) satisfies DataItem[];
+
+    const items = (await Promise.all(
+        itemList.map(
+            (item) =>
+                cache.tryGet(item.link, async () => {
+                    const resp = await ofetch(item.link);
+                    const $ = load(resp);
+                    item.description = $('#main section.article-section .doc-content > div').html() || item.description;
+                    return item;
+                }) as Promise<DataItem>
+        )
+    )) satisfies DataItem[];
 
     return {
         title: '游研社-' + $('title').text(),

package/lib/routes/zaobao/util.ts

@@ -38,7 +38,7 @@ const parseList = async (
 }> => {
     const response = await got_ins.get(baseUrl + sectionUrl);
     const $ = load(response.data);
-    let data = /realtime/.test(sectionUrl) ? $('.on-listing-pages') : $('.article-list').find('.article-type');
+    let data = $('.card-listing .card');
     if (data.length === 0) {
         // for HK version
         data = $('.clearfix').find('.list-block');
@@ -104,48 +104,7 @@ const parseList = async (
 
                 const articleBody = articleBodyNode.html();
 
-                const imageDataArray = [];
-                if ($1('.inline-figure-img').length) {
-                    // for SG version
-                    imageDataArray.push({
-                        type: 'normalHTML',
-                        html: $1('.inline-figure-img')
-                            .html()
-                            .replace(/\/\/.*\.com\/s3fs-public/, '//static.zaobao.com/s3fs-public')
-                            .replace(/s3\/files/, 's3fs-public'),
-                    });
-                }
-                if ($1('.body-content .loadme picture img').length) {
-                    // Unused?
-                    imageDataArray.push({
-                        type: 'data',
-                        src: $1('.body-content .loadme picture source')
-                            .attr('data-srcset')
-                            .replace(/\/\/.*\.com\/s3fs-public/, '//static.zaobao.com/s3fs-public')
-                            .replace(/s3\/files/, 's3fs-public'),
-                        title: $1('.body-content .loadme picture img').attr('title'),
-                    });
-                }
-                if ($1('.inline-figure-gallery').length) {
-                    // for SG version
-                    imageDataArray.push({
-                        type: 'normalHTML',
-                        html: $1('.inline-figure-gallery')
-                            .html()
-                            .replaceAll(/\/\/.*\.com\/s3fs-public/g, '//static.zaobao.com/s3fs-public')
-                            .replaceAll('s3/files', 's3fs-public'),
-                    });
-                }
-                if ($1('#carousel-article').length) {
-                    // for HK version, HK version of multi images use same selector as single image, so g is needed for all pages
-                    imageDataArray.push({
-                        type: 'normalHTML',
-                        html: $1('#carousel-article .carousel-inner')
-                            .html()
-                            .replaceAll(/\/\/.*\.com\/s3fs-public/g, '//static.zaobao.com/s3fs-public')
-                            .replaceAll('s3/files', 's3fs-public'),
-                    });
-                }
+                const imageDataArray = processImageData($1);
 
                 return {
                     // <- for SG version  -> for HK version
@@ -182,4 +141,43 @@ const orderContent = (parent) => {
     }
 };
 
+interface ImageData {
+    type: string;
+    html: string;
+    src?: string;
+    title?: string;
+}
+
+const processImageData = ($1) => {
+    const imageDataArray: ImageData[] = [];
+
+    const imageSelectors = [
+        '.inline-figure-img', // for SG version
+        '.body-content .loadme picture img', // Unused?
+        '.inline-figure-gallery', // for SG version
+        '#carousel-article', // for HK version, HK version of multi images use same selector as single image, so g is needed for all pages
+    ];
+
+    for (const selector of imageSelectors) {
+        if ($1(selector).length) {
+            let html = $1(selector === '#carousel-article' ? '#carousel-article .carousel-inner' : selector).html();
+
+            if (html) {
+                html = html.replaceAll(/\/\/.*\.com\/s3fs-public/g, '//static.zaobao.com/s3fs-public').replaceAll('s3/files', 's3fs-public');
+
+                imageDataArray.push({
+                    type: selector === '.body-content .loadme picture img' ? 'data' : 'normalHTML',
+                    html,
+                    ...(selector === '.body-content .loadme picture img' && {
+                        src: $1('.body-content .loadme picture source').attr('data-srcset'),
+                        title: $1(selector).attr('title'),
+                    }),
+                });
+            }
+        }
+    }
+
+    return imageDataArray;
+};
+
 export { parseList, orderContent };

package/lib/routes/zhihu/activities.ts

@@ -2,6 +2,7 @@ import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { header, processImage, getSignedHeader } from './utils';
 import { parseDate } from '@/utils/parse-date';
+import sanitizeHtml from 'sanitize-html';
 
 export const route: Route = {
     path: '/people/activities/:id',
@@ -75,7 +76,7 @@ async function handler(ctx) {
                     url = `https://zhuanlan.zhihu.com/p/${detail.id}`;
                     break;
                 case 'pin':
-                    title = detail.excerpt_title;
+                    title = sanitizeHtml(detail.excerpt_title);
                     author = detail.author.name;
                     for (const contentItem of detail.content) {
                         switch (contentItem.type) {

package/lib/routes/zhihu/answers.ts

@@ -1,7 +1,7 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
-import { header, processImage } from './utils';
+import { getCookieValueByKey, header, processImage } from './utils';
 import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
@@ -29,9 +29,13 @@ export const route: Route = {
 
 async function handler(ctx) {
     const id = ctx.req.param('id');
+
+    const zc0 = getCookieValueByKey('z_c0');
+
     const headers = {
         'User-Agent': 'ZhihuHybrid com.zhihu.android/Futureve/6.59.0 Mozilla/5.0 (Linux; Android 10; GM1900 Build/QKQ1.190716.003; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/85.0.4183.127 Mobile Safari/537.36',
         Referer: `https://www.zhihu.com/people/${id}/answers`,
+        Cookie: zc0 ? `z_c0=${zc0}` : '',
     };
 
     const response = await got({

package/lib/routes/zhihu/execlib/x-zse-96-v3.ts

@@ -2,6 +2,8 @@
 // https://blog.csdn.net/zjq592767809/article/details/126512798
 // https://blog.csdn.net/zhoumi_/article/details/126659351
 
+/* eslint-disable @typescript-eslint/no-unused-expressions */
+
 function i(e, t, n) {
     (t[n] = 255 & (e >>> 24)), (t[n + 1] = 255 & (e >>> 16)), (t[n + 2] = 255 & (e >>> 8)), (t[n + 3] = 255 & e);
 }

package/lib/routes/zhihu/timeline.ts

@@ -60,10 +60,13 @@ async function handler(ctx) {
                 const questionId = e.target.question.id;
                 return `${urlBase}/question/${questionId}/answer/${id}`;
             }
+            case 'pin':
             case 'article':
                 return e.target.url;
             case 'question':
                 return `${urlBase}/question/${id}`;
+            default:
+                return;
         }
         return '';
     };
@@ -111,11 +114,12 @@ async function handler(ctx) {
             link,
             author: e.target.author ? e.target.author.name : '',
             guid: link,
+            category: [e.verb],
         };
     };
 
     const out = feeds
-        .filter((e) => e.verb && e.verb !== 'MEMBER_VOTEUP_ARTICLE' && e.verb !== 'MEMBER_VOTEUP_ANSWER')
+        .filter((e) => e.verb)
         .map((e) => {
             if (e && e.type && e.type === 'feed_group') {
                 // A feed group contains a list of feeds whose structure is the same as a single feed
@@ -134,6 +138,7 @@ async function handler(ctx) {
                     description,
                     pubDate,
                     guid,
+                    category: [e.verb],
                 };
             }
             return buildItem(e);

package/lib/routes/zhihu/utils.ts

@@ -56,6 +56,13 @@ export const processImage = (content: string) => {
     return $.html();
 };
 
+export const getCookieValueByKey = (key: string) =>
+    config.zhihu.cookies
+        ?.split(';')
+        .map((e) => e.trim())
+        .find((e) => e.startsWith(key + '='))
+        ?.slice(key.length + 1);
+
 export const getSignedHeader = async (url: string, apiPath: string) => {
     // Because the API of zhihu.com has changed, we must use the value of `d_c0` (extracted from cookies) to calculate
     // `x-zse-96`. So first get `d_c0`, then get the actual data of a ZhiHu question. In this way, we don't need to
@@ -63,8 +70,12 @@ export const getSignedHeader = async (url: string, apiPath: string) => {
 
     // fisrt: get cookie(dc_0) from zhihu.com
     const dc0 = await cache.tryGet('zhihu:cookies:d_c0', async () => {
-        const response1 = await ofetch.raw(url);
-        const zseCk = response1._data.match(/var e="__zse_ck",t=\(typeof __g\.ck == 'string' && __g\.ck\)\|\|"([\w+/=]*?)",_=6048e5;/)?.[1];
+        if (getCookieValueByKey('d_c0')) {
+            return getCookieValueByKey('d_c0');
+        }
+        const response1 = await ofetch.raw('https://static.zhihu.com/zse-ck/v3.js');
+        const script = await response1._data.text();
+        const zseCk = script.match(/__g\.ck\|\|"([\w+/=\\]*?)",_=/)?.[1];
 
         const response2 = zseCk
             ? await ofetch.raw(url, {
@@ -95,11 +106,8 @@ export const getSignedHeader = async (url: string, apiPath: string) => {
     const f = `${xzse93}+${apiPath}+${dc0}`;
     const xzse96 = '2.0_' + g_encrypt(md5(f));
 
-    const zc0 = config.zhihu.cookies
-        ?.split(';')
-        .map((e) => e.trim())
-        .find((e) => e.includes('z_c0'))
-        ?.slice('z_c0='.length);
+    const zc0 = getCookieValueByKey('z_c0');
+
     return {
         cookie: `d_c0=${dc0}${zc0 ? `;z_c0=${zc0}` : ''}`,
         'x-zse-96': xzse96,

package/lib/routes/zhihu/zhuanlan.ts

@@ -29,12 +29,19 @@ export const route: Route = {
 
 async function handler(ctx) {
     const id = ctx.req.param('id');
+    // 知乎专栏链接存在两种格式, 一种以 'zhuanlan.' 开头, 另一种新增的以 'c_' 结尾
+    let url = `https://zhuanlan.zhihu.com/${id}`;
+    if (id.search('c_') === 0) {
+        url = `https://www.zhihu.com/column/${id}`;
+    }
 
+    const signedHeader = await getSignedHeader(url, `https://www.zhihu.com/api/v4/columns/${id}/items`);
     const listRes = await got({
         method: 'get',
         url: `https://www.zhihu.com/api/v4/columns/${id}/items`,
         headers: {
             ...header,
+            ...signedHeader,
             Referer: `https://zhuanlan.zhihu.com/${id}`,
         },
     });
@@ -44,19 +51,13 @@ async function handler(ctx) {
         url: `https://www.zhihu.com/api/v4/columns/${id}/pinned-items`,
         headers: {
             ...header,
+            ...signedHeader,
             Referer: `https://zhuanlan.zhihu.com/${id}`,
         },
     });
 
     listRes.data.data = [...listRes.data.data, ...pinnedRes.data.data];
 
-    // 知乎专栏链接存在两种格式, 一种以 'zhuanlan.' 开头, 另一种新增的以 'c_' 结尾
-    let url = `https://zhuanlan.zhihu.com/${id}`;
-    if (id.search('c_') === 0) {
-        url = `https://www.zhihu.com/column/${id}`;
-    }
-
-    const signedHeader = await getSignedHeader(url, `https://www.zhihu.com/api/v4/columns/${id}/items`);
     const infoRes = await got(url, {
         headers: {
             ...signedHeader,

package/lib/routes/zsxq/types.ts

@@ -63,6 +63,13 @@ export interface QATopic extends BasicTopic {
     question: {
         images?: TopicImage[];
         text?: string;
+        owner?: {
+            avatar_url: string;
+            description: string;
+            location: string;
+            name: string;
+            user_id: number;
+        };
     };
 }
 
@@ -140,6 +147,8 @@ export interface TopicImage {
 
 export type Topic = TalkTopic | QATopic | TaskTopic | SolutionTopic;
 
+export type ResponseData = UserInfo | GroupInfo | Topic[];
+
 export type UserInfoResponse = BasicResponse<UserInfo>;
 
 export type GroupInfoResponse = BasicResponse<GroupInfo>;

package/lib/routes/zsxq/utils.ts

@@ -1,10 +1,10 @@
 import got from '@/utils/got';
-import type { TopicImage, Topic, BasicResponse } from './types';
+import type { TopicImage, Topic, BasicResponse, ResponseData } from './types';
 import { parseDate } from '@/utils/parse-date';
 import { config } from '@/config';
 import type { DataItem } from '@/types';
 
-export async function customFetch<T extends BasicResponse<any>>(path: string, retryCount = 0): Promise<T['resp_data']> {
+export async function customFetch<T extends BasicResponse<ResponseData>>(path: string, retryCount = 0): Promise<T['resp_data']> {
     const apiUrl = 'https://api.zsxq.com/v2';
 
     const response = await got(apiUrl + path, {
@@ -46,8 +46,9 @@ export function generateTopicDataItem(topics: Topic[]): DataItem[] {
             case 'q&a':
                 title = topic.question?.text?.split('\n')[0] ?? '问答';
                 description = parseTopicContent(topic.question?.text, topic.question?.images);
+                description = `<blockquote>${String(topic.question?.owner?.name ?? '匿名用户')} 提问：${description}</blockquote>`;
                 if (topic.answered) {
-                    description += '<br><br>';
+                    description += '<br>' + topic.answer?.owner.name + ' 回答：<br><br>';
                     description += parseTopicContent(topic.answer?.text, topic.answer?.images);
                 }
                 break;

package/lib/setup.test.ts

@@ -31,7 +31,7 @@ const server = setupServer(
             choices: [
                 {
                     message: {
-                        content: 'Summary of the article.',
+                        content: 'AI processed content.',
                     },
                 },
             ],

package/lib/types.ts

@@ -286,3 +286,9 @@ export type RadarItem = {
               document: Document
           ) => string);
 };
+
+export type RadarDomain = {
+    _name: string;
+} & {
+    [subdomain: string]: RadarItem[];
+};

package/lib/utils/cache/redis.ts

@@ -64,7 +64,7 @@ export default {
         }
         if (key) {
             if (maxAge !== config.cache.contentExpire) {
-                // Only set cacheTtlKey if maxAge !== contentExpire
+                // intentionally store the cache ttl if it is not the default value
                 clients.redisClient.set(getCacheTtlKey(key), maxAge, 'EX', maxAge);
             }
             return clients.redisClient.set(key, value, 'EX', maxAge); // setMode: https://redis.io/commands/set