rsshub 1.0.0-master.ff3a9ce → 1.0.0-master.ff3ea5a

package/lib/routes/xwenming/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '未知文明',
+    url: 'xwenming.com',
+    categories: ['new-media'],
+    description: '',
+    lang: 'zh-CN',
+};

package/lib/routes/yahoo/news/utils.ts

@@ -84,7 +84,7 @@ const parseItem = (item, tryGet) =>
 
         const ldJson = JSON.parse($('script[type="application/ld+json"]').first().text());
         const author = `${$('span.caas-author-byline-collapse').text()} @${$('span.caas-attr-provider').text()}`;
-        const body = $('.caas-body');
+        const body = $('.atoms');
 
         body.find('noscript').remove();
         // remove padding

package/lib/routes/yande/post.ts

@@ -30,6 +30,9 @@ export const route: Route = {
 | ------- | -------- | ------- | -------- |
 | 1d | 1w | 1m | 1y |`,
     handler,
+    features: {
+        nsfw: true,
+    },
 };
 
 async function handler(ctx) {

package/lib/routes/ymgal/game.ts

@@ -23,6 +23,7 @@ export const route: Route = {
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
+        nsfw: true,
     },
     radar: [
         {

package/lib/routes/youtube/api/google.ts

@@ -8,6 +8,9 @@ import ofetch from '@/utils/ofetch';
 import * as cheerio from 'cheerio';
 import NotFoundError from '@/errors/types/not-found';
 import { Data } from '@/types';
+import dayjs from 'dayjs';
+import duration from 'dayjs/plugin/duration.js';
+dayjs.extend(duration);
 
 let count = 0;
 const youtube = {};
@@ -102,6 +105,8 @@ export const getDataByUsername = async ({ username, embed, filterShorts }: { use
     if (!playlistItems) {
         throw new NotFoundError("This channel doesn't have any content.");
     }
+    const videoIds = playlistItems.data.items.map((item) => item.snippet.resourceId.videoId).join(',');
+    const videoDetails = await utils.getVideos(videoIds, 'contentDetails', cache);
 
     return {
         title: `${userHandleData?.channelName || username} - YouTube`,
@@ -114,6 +119,7 @@ export const getDataByUsername = async ({ username, embed, filterShorts }: { use
                 const snippet = item.snippet;
                 const videoId = snippet.resourceId.videoId;
                 const img = utils.getThumbnail(snippet.thumbnails);
+                const detail = videoDetails?.data.items.find((d) => d.id === videoId);
                 return {
                     title: snippet.title,
                     description: utils.renderDescription(embed, videoId, img, utils.formatDescription(snippet.description)),
@@ -125,6 +131,7 @@ export const getDataByUsername = async ({ username, embed, filterShorts }: { use
                         {
                             url: getVideoUrl(videoId),
                             mime_type: 'text/html',
+                            duration_in_seconds: detail?.contentDetails.duration ? dayjs.duration(detail.contentDetails.duration).asSeconds() : undefined,
                         },
                     ],
                 };
@@ -140,6 +147,8 @@ export const getDataByChannelId = async ({ channelId, embed, filterShorts }: { c
     const playlistId = filterShorts ? utils.getPlaylistWithShortsFilter(channelId) : originalPlaylistId;
 
     const data = (await utils.getPlaylistItems(playlistId, 'snippet', cache)).data.items;
+    const videoIds = data.map((item) => item.snippet.resourceId.videoId).join(',');
+    const videoDetails = await utils.getVideos(videoIds, 'contentDetails', cache);
 
     return {
         title: `${data[0].snippet.channelTitle} - YouTube`,
@@ -151,6 +160,7 @@ export const getDataByChannelId = async ({ channelId, embed, filterShorts }: { c
                 const snippet = item.snippet;
                 const videoId = snippet.resourceId.videoId;
                 const img = utils.getThumbnail(snippet.thumbnails);
+                const detail = videoDetails?.data.items.find((d) => d.id === videoId);
                 return {
                     title: snippet.title,
                     description: utils.renderDescription(embed, videoId, img, utils.formatDescription(snippet.description)),
@@ -158,6 +168,13 @@ export const getDataByChannelId = async ({ channelId, embed, filterShorts }: { c
                     link: `https://www.youtube.com/watch?v=${videoId}`,
                     author: snippet.videoOwnerChannelTitle,
                     image: img.url,
+                    attachments: [
+                        {
+                            url: getVideoUrl(videoId),
+                            mime_type: 'text/html',
+                            duration_in_seconds: detail?.contentDetails.duration ? dayjs.duration(detail.contentDetails.duration).asSeconds() : undefined,
+                        },
+                    ],
                 };
             }),
     };
@@ -167,6 +184,8 @@ export const getDataByPlaylistId = async ({ playlistId, embed }: { playlistId: s
     const playlistTitle = (await utils.getPlaylist(playlistId, 'snippet', cache)).data.items[0].snippet.title;
 
     const data = (await utils.getPlaylistItems(playlistId, 'snippet', cache)).data.items.filter((d) => d.snippet.title !== 'Private video' && d.snippet.title !== 'Deleted video');
+    const videoIds = data.map((item) => item.snippet.resourceId.videoId).join(',');
+    const videoDetails = await utils.getVideos(videoIds, 'contentDetails', cache);
 
     return {
         title: `${playlistTitle} by ${data[0].snippet.channelTitle} - YouTube`,
@@ -176,6 +195,7 @@ export const getDataByPlaylistId = async ({ playlistId, embed }: { playlistId: s
             const snippet = item.snippet;
             const videoId = snippet.resourceId.videoId;
             const img = utils.getThumbnail(snippet.thumbnails);
+            const detail = videoDetails?.data.items.find((d) => d.id === videoId);
             return {
                 title: snippet.title,
                 description: utils.renderDescription(embed, videoId, img, utils.formatDescription(snippet.description)),
@@ -183,6 +203,13 @@ export const getDataByPlaylistId = async ({ playlistId, embed }: { playlistId: s
                 link: `https://www.youtube.com/watch?v=${videoId}`,
                 author: snippet.videoOwnerChannelTitle,
                 image: img.url,
+                attachments: [
+                    {
+                        url: getVideoUrl(videoId),
+                        mime_type: 'text/html',
+                        duration_in_seconds: detail?.contentDetails.duration ? dayjs.duration(detail.contentDetails.duration).asSeconds() : undefined,
+                    },
+                ],
             };
         }),
     };

package/lib/routes/youtube/api/youtubei.ts

@@ -1,3 +1,4 @@
+import { getSubtitles } from 'youtube-caption-extractor';
 import cache from '@/utils/cache';
 import { Innertube } from 'youtubei.js';
 import utils, { getVideoUrl } from '../utils';
@@ -6,6 +7,38 @@ import { parseRelativeDate } from '@/utils/parse-date';
 
 const innertubePromise = Innertube.create();
 
+function pad(n: number, width: number = 2) {
+    return String(n).padStart(width, '0');
+}
+
+function toSrtTime(seconds: number): string {
+    const totalMs = Math.floor(seconds * 1000);
+    const hours = Math.floor(totalMs / 3_600_000);
+    const minutes = Math.floor((totalMs % 3_600_000) / 60000);
+    const secs = Math.floor((totalMs % 60000) / 1000);
+    const millis = totalMs % 1000;
+    return `${pad(hours)}:${pad(minutes)}:${pad(secs)},${pad(millis, 3)}`;
+}
+
+type Subtitle = {
+    start: string;
+    dur: string;
+    text: string;
+};
+
+function convertToSrt(segments: Subtitle[]): string {
+    return segments
+        .map((seg, index) => {
+            const start = Number.parseFloat(seg.start);
+            const end = start + Number.parseFloat(seg.dur);
+            return `${index + 1}
+${toSrtTime(start)} --> ${toSrtTime(end)}
+${seg.text}
+`;
+        })
+        .join('\n');
+}
+
 export const getChannelIdByUsername = (username: string) =>
     cache.tryGet(`youtube:getChannelIdByUsername:${username}`, async () => {
         const innertube = await innertubePromise;
@@ -18,6 +51,13 @@ export const getDataByUsername = async ({ username, embed, filterShorts }: { use
     return getDataByChannelId({ channelId, embed, filterShorts });
 };
 
+const getSubtitlesByVideoId = (videoId: string) =>
+    cache.tryGet(`youtube:getSubtitlesByVideoId:${videoId}`, async () => {
+        const subtitles = await getSubtitles({ videoID: videoId });
+        const srt = convertToSrt(subtitles);
+        return srt;
+    });
+
 export const getDataByChannelId = async ({ channelId, embed }: { channelId: string; embed: boolean; filterShorts: boolean }): Promise<Data> => {
     const innertube = await innertubePromise;
     const channel = await innertube.getChannel(channelId);
@@ -29,27 +69,37 @@ export const getDataByChannelId = async ({ channelId, embed }: { channelId: stri
         image: channel.metadata.avatar?.[0].url,
         description: channel.metadata.description,
 
-        item: videos.videos
-            .filter((video) => 'video_id' in video)
-            .map((video) => {
-                const img = 'best_thumbnail' in video ? video.best_thumbnail?.url : ('thumbnails' in video ? video.thumbnails?.[0]?.url : undefined);
+        item: await Promise.all(
+            videos.videos
+                .filter((video) => 'video_id' in video)
+                .map(async (video) => {
+                    const srt = await getSubtitlesByVideoId(video.video_id);
+                    const dataUrl = `data:text/plain;charset=utf-8,${encodeURIComponent(srt)}`;
 
-                return {
-                    title: video.title.text || `YouTube Video ${video.video_id}`,
-                    description: 'description_snippet' in video ? utils.renderDescription(embed, video.video_id, img, utils.formatDescription(video.description_snippet?.toHTML())) : null,
-                    link: `https://www.youtube.com/watch?v=${video.video_id}`,
-                    author: typeof video.author === 'string' ? video.author : (video.author.name === 'N/A' ? undefined : video.author.name),
-                    image: img,
-                    pubDate: 'published' in video && video.published?.text ? parseRelativeDate(video.published.text) : undefined,
-                    attachments: [
-                        {
-                            url: getVideoUrl(video.video_id),
-                            mime_type: 'text/html',
-                            duration_in_seconds: video.duration && 'seconds' in video.duration ? video.duration.seconds : undefined,
-                        },
-                    ],
-                };
-            }),
+                    const img = 'best_thumbnail' in video ? video.best_thumbnail?.url : 'thumbnails' in video ? video.thumbnails?.[0]?.url : undefined;
+
+                    return {
+                        title: video.title.text || `YouTube Video ${video.video_id}`,
+                        description: 'description_snippet' in video ? utils.renderDescription(embed, video.video_id, img, utils.formatDescription(video.description_snippet?.toHTML())) : null,
+                        link: `https://www.youtube.com/watch?v=${video.video_id}`,
+                        author: typeof video.author === 'string' ? video.author : video.author.name === 'N/A' ? undefined : video.author.name,
+                        image: img,
+                        pubDate: 'published' in video && video.published?.text ? parseRelativeDate(video.published.text) : undefined,
+                        attachments: [
+                            {
+                                url: getVideoUrl(video.video_id),
+                                mime_type: 'text/html',
+                                duration_in_seconds: video.duration && 'seconds' in video.duration ? video.duration.seconds : undefined,
+                            },
+                            {
+                                url: dataUrl,
+                                mime_type: 'text/srt',
+                                title: 'Subtitles',
+                            },
+                        ],
+                    };
+                })
+        ),
     };
 };
 

package/lib/routes/youtube/community.ts

@@ -35,7 +35,9 @@ async function handler(ctx) {
 
     const channelMetadata = ytInitialData.metadata.channelMetadataRenderer;
     const username = channelMetadata.title;
-    const communityTab = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.find((tab) => tab.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url.endsWith('/community'));
+    const communityTab = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.find(
+        (tab) => tab.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url.endsWith('/posts') || tab.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url.endsWith('/community')
+    );
     const list = communityTab.tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents;
 
     if (list[0].messageRenderer) {

package/lib/routes/youtube/utils.ts

@@ -50,18 +50,18 @@ export const getChannelWithUsername = (username, part, cache) =>
         );
         return res;
     });
-// not in use
-// export const getVideoAuthor = async (id, part) => {
-//     const res = await exec((youtube) =>
-//         youtube.videos.list({
-//             part,
-//             id,
-//         })
-//     );
-//     return res;
-// }
+export const getVideos = (id, part, cache) =>
+    cache.tryGet(`youtube:getVideos:${id}`, async () => {
+        const res = await exec((youtube) =>
+            youtube.videos.list({
+                part,
+                id,
+            })
+        );
+        return res;
+    });
 export const getThumbnail = (thumbnails) => thumbnails.maxres || thumbnails.standard || thumbnails.high || thumbnails.medium || thumbnails.default;
-export const formatDescription = (description) => description.replaceAll(/\r\n|\r|\n/g, '<br>');
+export const formatDescription = (description) => description?.replaceAll(/\r\n|\r|\n/g, '<br>');
 export const renderDescription = (embed, videoId, img, description) =>
     art(path.join(__dirname, 'templates/description.art'), {
         embed,
@@ -93,7 +93,9 @@ export async function getSubscriptionsRecusive(part, nextPageToken?) {
     // recursively get next page
     if (res.data.nextPageToken) {
         const next = await getSubscriptionsRecusive(part, res.data.nextPageToken);
-        res.data.items = [...res.data.items, ...next.data.items];
+        if (next.data.items) {
+            res.data.items = [...(res.data.items || []), ...next.data.items];
+        }
     }
     return res;
 }
@@ -140,11 +142,12 @@ export const callApi = async <T>({ googleApi, youtubeiApi, params }: { googleApi
     return await youtubeiApi(params);
 };
 
-const youtubeUtils = {
+export default {
     getPlaylistItems,
     getPlaylist,
     getChannelWithId,
     getChannelWithUsername,
+    getVideos,
     getThumbnail,
     formatDescription,
     renderDescription,
@@ -155,4 +158,3 @@ const youtubeUtils = {
     getVideoUrl,
     getPlaylistWithShortsFilter,
 };
-export default youtubeUtils;

package/lib/routes/zaobao/util.ts

@@ -35,10 +35,10 @@ const parseList = async (
 }> => {
     const response = await got_ins.get(baseUrl + sectionUrl);
     const $ = load(response.data);
-    let data = $('.card-listing .card');
+    let data = $('.card-listing .card .content-header a');
     if (data.length === 0) {
         // for HK version
-        data = $('.clearfix').find('.list-block');
+        data = $('[data-testid="article-list"] article a.article-link');
     }
 
     const title = $('meta[property="og:title"]').attr('content');
@@ -47,7 +47,7 @@ const parseList = async (
         data.toArray().map((item) => {
             const $item = $(item);
             // addBack: for HK version
-            let link = $item.find('a').addBack('a')[0].attribs.href;
+            let link = $item.attr('href');
 
             if (link[0] !== '/') {
                 // https://www.zaobao.com/interactive-graphics
@@ -88,7 +88,7 @@ const parseList = async (
                     // HK
                     title = $1('h1.article-title').text();
                     const jsonText = $1("head script[type='application/ld+json']")
-                        .eq(1)
+                        .eq(0)
                         .text()
                         .replaceAll(/[\u0000-\u001F\u007F-\u009F]/g, '');
                     time = new Date(JSON.parse(jsonText)?.datePublished);
@@ -106,11 +106,14 @@ const parseList = async (
                 $1('#video-freemium-player').remove();
                 $1('script').remove();
                 $1('.bff-google-ad').remove();
+                $1('.bff-recommend-article').remove();
 
                 let articleBodyNode = $1('.articleBody');
                 if (articleBodyNode.length === 0) {
                     // for HK version
-                    orderContent($1('.article-body'));
+                    $1('.read-on-app-cover').remove();
+                    $1('astro-island').remove();
+                    $1('.further-reading').remove();
                     articleBodyNode = $1('.article-body');
                 }
 

package/lib/routes/zhihu/activities.ts

@@ -103,6 +103,9 @@ async function handler(ctx) {
                                 src="${contentItem.playlist[1].url}"></video></p>`;
 
                                 break;
+                            case 'link_card':
+                                link = `<p><a href="${contentItem.url.split('?')[0]}" target="_blank"></a></p>`;
+                                break;
 
                             default:
                                 throw new Error(`Unknown type: ${contentItem.type}`);

package/lib/routes/zhihu/execlib/x-zse-96-v3.ts

@@ -5,7 +5,7 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions */
 
 function i(e, t, n) {
-    (t[n] = 255 & (e >>> 24)), (t[n + 1] = 255 & (e >>> 16)), (t[n + 2] = 255 & (e >>> 8)), (t[n + 3] = 255 & e);
+    ((t[n] = 255 & (e >>> 24)), (t[n + 1] = 255 & (e >>> 16)), (t[n + 2] = 255 & (e >>> 8)), (t[n + 3] = 255 & e));
 }
 function B(e, t) {
     return ((255 & e[t]) << 24) | ((255 & e[t + 1]) << 16) | ((255 & e[t + 2]) << 8) | (255 & e[t + 3]);
@@ -16,7 +16,7 @@ function Q(e, t) {
 function G(e) {
     const t = Array.from({ length: 4 });
     const n = Array.from({ length: 4 });
-    i(e, t, 0), (n[0] = h.zb[255 & t[0]]), (n[1] = h.zb[255 & t[1]]), (n[2] = h.zb[255 & t[2]]), (n[3] = h.zb[255 & t[3]]);
+    (i(e, t, 0), (n[0] = h.zb[255 & t[0]]), (n[1] = h.zb[255 & t[1]]), (n[2] = h.zb[255 & t[2]]), (n[3] = h.zb[255 & t[3]]));
     const r = B(n, 0);
     return r ^ Q(r, 2) ^ Q(r, 10) ^ Q(r, 18) ^ Q(r, 24);
 }
@@ -29,19 +29,19 @@ const __g = {
                 a[c] = o[c] ^ t[c];
             }
             // eslint-disable-next-line unicorn/prefer-spread
-            (t = __g.r(a)), (n = n.concat(t)), i++;
+            ((t = __g.r(a)), (n = n.concat(t)), i++);
         }
         return n;
     },
     r(e) {
         const t = Array.from({ length: 16 });
         const n = Array.from({ length: 36 });
-        (n[0] = B(e, 0)), (n[1] = B(e, 4)), (n[2] = B(e, 8)), (n[3] = B(e, 12));
+        ((n[0] = B(e, 0)), (n[1] = B(e, 4)), (n[2] = B(e, 8)), (n[3] = B(e, 12)));
         for (let r = 0; r < 32; r++) {
             const o = G(n[r + 1] ^ n[r + 2] ^ n[r + 3] ^ h.zk[r]);
             n[r + 4] = n[r] ^ o;
         }
-        return i(n[35], t, 0), i(n[34], t, 4), i(n[33], t, 8), i(n[32], t, 12), t;
+        return (i(n[35], t, 0), i(n[34], t, 4), i(n[33], t, 8), i(n[32], t, 12), t);
     },
 };
 const h = {

package/lib/routes/zhizhuan100/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '智篆商业',
+    url: 'www.zhizhuan100.com.cn',
+    lang: 'zh-CN',
+};

package/lib/routes/zhizhuan100/report.ts

@@ -0,0 +1,79 @@
+import { Route, ViewType } from '@/types';
+import { parseRelativeDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch';
+import * as cheerio from 'cheerio';
+
+export const route: Route = {
+    path: '/analytic',
+    categories: ['finance'],
+    view: ViewType.Articles,
+    example: '/zhizhuan100/analytic',
+    radar: [
+        {
+            source: ['www.zhizhuan100.com.cn/analysis'],
+        },
+    ],
+    name: 'analytic',
+    maintainers: ['Cedaric'],
+    handler,
+};
+
+async function handler() {
+    const urlData = await ofetch('https://www.zhizhuan100.com.cn/analysis');
+
+    const $ = cheerio.load(urlData);
+
+    const bodyJsUrl: string | undefined = $('script[src*="Body.js"]').attr('src');
+
+    if (!bodyJsUrl) {
+        throw new Error('无法找到 Body.js 脚本文件');
+    }
+
+    const responseData = await ofetch(bodyJsUrl, {
+        parseResponse: (txt) => txt,
+    });
+
+    const htmlMatch = responseData.match(/document\.write\('(.*)'\);/s);
+    if (!htmlMatch) {
+        throw new Error('无法找到HTML内容');
+    }
+
+    const htmlContent = JSON.parse(`"${htmlMatch[1]}"`);
+    const $content = cheerio.load(htmlContent);
+
+    const listItems = $content('.w-list-item');
+
+    const items = listItems
+        .toArray()
+        .map((item) => {
+            const $item = $content(item);
+            const titleElement = $item.find('.w-list-title');
+            const dateElement = $item.find('.w-list-date');
+            const linkElement = $item.find('.w-list-link');
+            const imgElement = $item.find('.w-listpic-in');
+
+            const title = titleElement.text().trim() || '';
+            const dateText = dateElement.text().trim() || '';
+            const href = linkElement.attr('href') || '';
+            const imgSrc = imgElement.attr('src') || '';
+
+            const idMatch = href.match(/\/productinfo\/(\d+)\.html/);
+            const id = idMatch ? idMatch[1] : '';
+
+            return {
+                title,
+                pubDate: parseRelativeDate(dateText),
+                link: href.startsWith('http') ? href : `https://www.zhizhuan100.com.cn${href}`,
+                description: `<img src="${imgSrc.startsWith('//') ? 'https:' + imgSrc : imgSrc}" alt="${title}">`,
+                guid: id,
+            };
+        })
+        .filter((item) => item.title);
+
+    return {
+        title: '智篆商业-消费报告',
+        link: 'https://www.zhizhuan100.com.cn/analysis',
+        allowEmpty: true,
+        item: items,
+    };
+}

package/lib/routes/zimuxia/portfolio.ts

@@ -69,7 +69,7 @@ async function handler(ctx) {
                 guid: `${currentUrl}#${title}`,
             };
         })
-        .reverse();
+        .toReversed();
 
     return {
         title: `${$('.content-page-title').text()} - FIX字幕侠`,