rsshub 1.0.0-master.fc1b99f → 1.0.0-master.fc2cf18

package/lib/routes/yamibo/utils.ts

@@ -0,0 +1,114 @@
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+import ofetch from '@/utils/ofetch';
+import { config } from '@/config';
+import asyncPool from 'tiny-async-pool';
+import { JSDOM } from 'jsdom';
+import type { Cheerio, Element } from 'cheerio';
+
+export const bbsOrigin = 'https://bbs.yamibo.com';
+
+export function getDate(date: string): Date {
+    return timezone(parseDate(date), 8);
+}
+
+export async function fetchThread(
+    tid: string,
+    options?: {
+        ordertype?: string;
+        _dsign?: string;
+    },
+    retry = 0
+): Promise<{
+    link: string;
+    data?: string;
+}> {
+    const { auth, salt } = config.yamibo;
+    const params = new URLSearchParams();
+    params.set('mod', 'viewthread');
+    params.set('tid', tid);
+    if (options?.ordertype) {
+        params.set('ordertype', options.ordertype);
+    }
+    if (options?._dsign) {
+        params.set('_dsign', options._dsign);
+    }
+    const link = `https://bbs.yamibo.com/forum.php?${params.toString()}`;
+
+    const headers: HeadersInit = {};
+
+    if (auth && salt) {
+        headers.cookie = `EeqY_2132_saltkey=${salt}; EeqY_2132_auth=${auth}`;
+    }
+
+    const data = await ofetch<string>(link, { headers });
+
+    // sometimes may trigger anti-crawling measures
+    if (data.startsWith('<script type="text/javascript">') && retry <= 3) {
+        let script = data.match(/<script type="text\/javascript">([\S\s]*?)<\/script>/)![1];
+        script = script.replace(/= location;|=location;/, '=fakeLocation;');
+        script = script.replace('location.replace', 'foo');
+        script = script.replace('location.assign', 'foo');
+        script = script.replace(/location\[[^\]]*]\(/, 'foo(');
+        script = script.replace(/location\[[^\]]*]=/, 'window.locationValue=');
+        script = script.replace('location.href=', 'window.locationValue=');
+        script = script.replace('location=', 'window.locationValue=');
+        const dom = new JSDOM(
+            `<script>
+                function foo(value) { window.locationValue = value; };
+                fakeLocation = { href: '', replace: foo, assign: foo };
+                Object.defineProperty(fakeLocation, 'href', {
+                    set: function (value) {
+                        window.locationValue = value;
+                    }
+                });
+                ${script}
+            </script>`,
+            {
+                runScripts: 'dangerously',
+            }
+        );
+        const locationValue = dom.window.locationValue;
+        if (locationValue) {
+            const searchParams = new URLSearchParams(locationValue);
+            const _dsign = searchParams.get('_dsign');
+            if (_dsign) {
+                options = {
+                    ...options,
+                    _dsign,
+                };
+            }
+        }
+        return await fetchThread(tid, options, ++retry);
+    }
+
+    return {
+        link,
+        data,
+    };
+}
+
+export function generateDescription($item: Cheerio<Element>, postId: string) {
+    const content = $item.find(`#postmessage_${postId}`).parent();
+    content.find('img').each((_, img) => {
+        const src = img.attribs.zoomfile ?? img.attribs.src;
+        img.attribs.src = `${bbsOrigin}/${src}`;
+    });
+    let description = content.html() ?? '';
+
+    const images = $item.find('.pattl img').toArray();
+    for (const img of images) {
+        const src = img.attribs.zoomfile ?? img.attribs.src;
+        description += `<img src="${bbsOrigin}/${src}" />`;
+    }
+
+    return description;
+}
+
+export async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
+    const results: Awaited<OUT[]> = [];
+    for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
+        results.push(result);
+    }
+    return results;
+}

package/lib/routes/yande/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'yande',
+    url: 'yande.re',
+    description: `yande post`,
+};

package/lib/{routes-deprecated/yande.re/post-popular-recent.js → routes/yande/post.ts}

@@ -1,8 +1,29 @@
-const got = require('@/utils/got');
-const queryString = require('query-string');
+import { Route } from '@/types';
+import got from '@/utils/got';
+import queryString from 'query-string';
 
-module.exports = async (ctx) => {
-    const { period = '1d' } = ctx.params;
+export const route: Route = {
+    path: '/post/popular_recent/:period?',
+    categories: ['anime'],
+    example: '/yande/post/popular_recent/1d',
+    parameters: {
+        period: '展示时间',
+    },
+    radar: [
+        {
+            source: ['yande.re/post/'],
+        },
+    ],
+    name: 'posts',
+    maintainers: ['fashioncj'],
+    description: `| 最近 24 小时    | 最近一周     | 最近一月    | 最近一年     | 
+  | ------- | -------- | ------- | -------- | 
+  | 1d | 1w | 1m ｜1y｜`,
+    handler,
+};
+
+async function handler(ctx) {
+    const { period = '1d' } = ctx.req.param();
 
     const response = await got({
         url: 'https://yande.re/post/popular_recent.json',
@@ -27,7 +48,7 @@ module.exports = async (ctx) => {
 
     const title = titles[period];
 
-    ctx.state.data = {
+    return {
         title: `${title} - yande.re`,
         link: `https://yande.re/post/popular_recent?period=${period}`,
         item: posts.map((post) => ({
@@ -60,4 +81,4 @@ module.exports = async (ctx) => {
             category: post.tags.split(/\s+/),
         })),
     };
-};
+}

package/lib/routes/ycwb/index.ts

@@ -8,7 +8,7 @@ import { load } from 'cheerio';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 export const route: Route = {
     path: '/:node',

package/lib/routes/yenpress/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Yen Press',
+    url: 'yenpress.com',
+    categories: ['reading'],
+};

package/lib/routes/yenpress/series.ts

@@ -0,0 +1,115 @@
+import { DataItem, Route } from '@/types';
+import type { Context } from 'hono';
+
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
+import * as cheerio from 'cheerio';
+import timezone from '@/utils/timezone';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import path from 'node:path';
+import { getCurrentPath } from '@/utils/helpers';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+const render = (data) => art(path.join(__dirname, 'templates', 'series.art'), data);
+
+export const route: Route = {
+    path: '/series/:name',
+    example: '/yenpress/series/alya-sometimes-hides-her-feelings-in-russian',
+    parameters: { name: 'Series name' },
+    name: 'Series',
+    maintainers: ['TonyRL'],
+    handler,
+    radar: [
+        {
+            source: ['yenpress.com/series/:name'],
+            target: '/series/:name',
+        },
+    ],
+};
+
+async function handler(ctx: Context) {
+    const { name: series } = ctx.req.param();
+    const baseUrl = 'https://yenpress.com';
+    const link = `https://yenpress.com/series/${series}`;
+
+    const response = await ofetch(link);
+    const $ = cheerio.load(response);
+
+    const list = $('.show-more-container .inline_block')
+        .toArray()
+        .map((item) => {
+            const $item = $(item);
+            return {
+                title: $item.find('span').text().trim(),
+                link: new URL($item.find('a').attr('href')!, baseUrl).href,
+            };
+        }) as DataItem[];
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link!, async () => {
+                const response = await ofetch(item.link!);
+                const $ = cheerio.load(response);
+
+                item.category = $('.detail-labels.mobile-only')
+                    .eq(0)
+                    .find('a')
+                    .toArray()
+                    .map((a) => $(a).text().trim());
+
+                $('svg, .social-share, .desktop-only, .detail-labels').remove();
+
+                const cover = $('.book-info').find('.book-cover-img').html();
+
+                const bookInfo = $('.buy-info .deliver')
+                    .toArray()
+                    .map((item, i) => ({
+                        deliver: $(item).text().trim(),
+                        price: $('.book-price').eq(i).text().trim(),
+                        from: $('.services')
+                            .eq(i)
+                            .find('.service')
+                            .toArray()
+                            .map((service) => {
+                                const a = $(service).find('a');
+                                return {
+                                    name: a.text().trim(),
+                                    link: a.attr('href'),
+                                };
+                            }),
+                        detail: $('.detail-info')
+                            .eq(i)
+                            .find('div span')
+                            .toArray()
+                            .map((span) => {
+                                const $span = $(span);
+                                return {
+                                    key: $span.text().trim(),
+                                    value: $span.next().text().trim(),
+                                };
+                            }),
+                    }));
+
+                const info = $('.book-info');
+                info.find('.buy-info, .series-cover').remove();
+
+                item.description = render({
+                    cover: cover! + info.html(),
+                    bookInfo,
+                });
+                item.pubDate = timezone(parseDate(bookInfo[0].detail.find((d) => d.key === 'Release Date')!.value), 0);
+
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: $('head title').text().trim(),
+        description: $('.social-share p').text().trim(),
+        link,
+        item: items,
+    };
+}

package/lib/routes/yenpress/templates/series.art

@@ -0,0 +1,25 @@
+{{ if cover }}
+    {{@ cover }}<br>
+{{ /if }}
+
+{{ if bookInfo }}
+    {{ each bookInfo info }}
+        <p>{{ info.deliver }}: {{ info.price }}</p>
+        Buy from:
+        <ul>
+            {{ each info.from f }}
+                <li><a href="{{ f.link }}">{{ f.name }}</a></li>
+            {{ /each }}
+        </ul>
+        <b>FULL DETAILS</b>
+        <table>
+            {{ each info.detail d }}
+                <tr>
+                    <td>{{ d.key }}</td>
+                    <td>{{ d.value }}</td>
+                </tr>
+            {{ /each }}
+        </table>
+        <br>
+    {{ /each }}
+{{ /if }}

package/lib/routes/ygkkk/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '甬哥侃侃侃YouTube教程摘要随笔',
+    url: 'ygkkk.blogspot.com',
+};

package/lib/routes/ygkkk/rss.ts

@@ -0,0 +1,48 @@
+import { Route } from '@/types';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+
+export const route: Route = {
+    path: '/',
+    categories: ['blog'],
+    example: '/ygkkk',
+    radar: [
+        {
+            source: ['ygkkk.blogspot.com/'],
+        },
+    ],
+    name: '最新发表',
+    maintainers: ['cnkmmk'],
+    handler,
+    url: 'ygkkk.blogspot.com/',
+};
+
+async function handler() {
+    const currentUrl = 'https://ygkkk.blogspot.com';
+    const response = await got(`${currentUrl}/feeds/posts/default?alt=rss`);
+    const $ = load(response.data, { xmlMode: true });
+    const title_main = $('channel > title').text();
+    const description_main = $('channel > description').text();
+    const items = $('channel > item')
+        .map((_, item) => {
+            const $item = $(item);
+            const link = $item.find('link').text();
+            const title = $item.find('title').text();
+            const description = $item.find('description').text();
+            const pubDate = $item.find('pubDate').text();
+            return {
+                link,
+                pubDate, // no need to normalize because it's from a valid RSS feed
+                title,
+                description,
+            };
+        })
+        .get();
+
+    return {
+        title: title_main,
+        description: description_main,
+        link: currentUrl,
+        item: items,
+    };
+}

package/lib/routes/yicai/dt.ts

@@ -7,7 +7,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 const columns = {
     article: 2,

package/lib/routes/yicai/utils.ts

@@ -6,7 +6,7 @@ import { load } from 'cheerio';
 import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 const rootUrl = 'https://www.yicai.com';
 

package/lib/routes/ymgal/article.ts

@@ -42,7 +42,7 @@ async function handler(ctx) {
         await Promise.all(
             Object.values(types).map(async (type) => {
                 const response = await got(`${host}/co/topic/list${type}`);
-                data.push(response.data.data);
+                data.push(...response.data.data);
             })
         );
         data = data.sort((a, b) => b.publishTime - a.publishTime).slice(0, 10);

package/lib/routes/ymgal/game.ts

@@ -4,7 +4,7 @@ const __dirname = getCurrentPath(import.meta.url);
 
 import got from '@/utils/got';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 import { load } from 'cheerio';
 
 const host = 'https://www.ymgal.games';

package/lib/routes/yoasobi-music/info.ts

@@ -6,7 +6,7 @@ import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
 import { parseJSONP } from './jsonp-helper';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 export const route: Route = {
     path: '/info/:category?',

package/lib/routes/yoasobi-music/jsonp-helper.ts

@@ -5,7 +5,7 @@ function parseJSONP(jsonpData) {
         let jsonString = jsonpData.substring(startPos + 1, endPos + 1);
 
         // remove escaped single quotes since they are not valid json
-        jsonString = jsonString.replaceAll("\\'", "'");
+        jsonString = jsonString.replaceAll(String.raw`\'`, "'");
 
         return JSON.parse(jsonString);
     } catch (error_) {

package/lib/routes/yoasobi-music/live.ts

@@ -5,7 +5,7 @@ const __dirname = getCurrentPath(import.meta.url);
 import got from '@/utils/got';
 import { parseJSONP } from './jsonp-helper';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 export const route: Route = {
     path: '/live',

package/lib/routes/yoasobi-music/media.ts

@@ -5,7 +5,7 @@ const __dirname = getCurrentPath(import.meta.url);
 import got from '@/utils/got';
 import { parseJSONP } from './jsonp-helper';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 export const route: Route = {
     path: '/media',

package/lib/routes/youku/channel.ts

@@ -6,7 +6,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 
 export const route: Route = {
     path: '/channel/:channelId/:embed?',

package/lib/routes/youtube/channel.ts

@@ -3,6 +3,8 @@ import cache from '@/utils/cache';
 import utils from './utils';
 import { config } from '@/config';
 import { parseDate } from '@/utils/parse-date';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
 
 export const route: Route = {
     path: '/channel/:id/:embed?',
@@ -10,7 +12,12 @@ export const route: Route = {
     example: '/youtube/channel/UCDwDMPOZfxVV0x_dz0eQ8KQ',
     parameters: { id: 'YouTube channel id', embed: 'Default to embed the video, set to any value to disable embedding' },
     features: {
-        requireConfig: false,
+        requireConfig: [
+            {
+                name: 'YOUTUBE_KEY',
+                description: ' YouTube API Key, support multiple keys, split them with `,`, [API Key application](https://console.developers.google.com/)',
+            },
+        ],
         requirePuppeteer: false,
         antiCrawler: false,
         supportBT: false,
@@ -23,7 +30,7 @@ export const route: Route = {
             target: '/channel/:id',
         },
     ],
-    name: 'Channel',
+    name: 'Channel with id',
     maintainers: ['DIYgod'],
     handler,
     description: `:::tip
@@ -33,13 +40,13 @@ YouTube provides official RSS feeds for channels, for instance [https://www.yout
 
 async function handler(ctx) {
     if (!config.youtube || !config.youtube.key) {
-        throw new Error('YouTube RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
+        throw new ConfigNotFoundError('YouTube RSS is disabled due to the lack of <a href="https://docs.rsshub.app/deploy/config#route-specific-configurations">relevant config</a>');
     }
     const id = ctx.req.param('id');
     const embed = !ctx.req.param('embed');
 
     if (!utils.isYouTubeChannelId(id)) {
-        throw new Error(`Invalid YouTube channel ID. \nYou may want to use <code>/youtube/user/:id</code> instead.`);
+        throw new InvalidParameterError(`Invalid YouTube channel ID. \nYou may want to use <code>/youtube/user/:id</code> instead.`);
     }
 
     const playlistId = (await utils.getChannelWithId(id, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads;
@@ -62,6 +69,7 @@ async function handler(ctx) {
                     pubDate: parseDate(snippet.publishedAt),
                     link: `https://www.youtube.com/watch?v=${videoId}`,
                     author: snippet.videoOwnerChannelTitle,
+                    image: img.url,
                 };
             }),
     };

package/lib/routes/youtube/charts.ts

@@ -1,6 +1,6 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { renderDescription } from './utils';
 import { config } from '@/config';
 
@@ -9,14 +9,6 @@ export const route: Route = {
     categories: ['social-media'],
     example: '/youtube/charts',
     parameters: { category: 'Chart, see table below, default to `TopVideos`', country: 'Country Code, see table below, default to global', embed: 'Default to embed the video, set to any value to disable embedding' },
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
     name: 'Music Charts',
     maintainers: ['TonyRL'],
     handler,
@@ -95,12 +87,13 @@ async function handler(ctx) {
     const { content } = await cache.tryGet(
         `youtube:charts:${country ?? 'global'}`,
         async () => {
-            const { data } = await got.post('https://charts.youtube.com/youtubei/v1/browse', {
-                searchParams: {
+            const data = await ofetch('https://charts.youtube.com/youtubei/v1/browse', {
+                method: 'POST',
+                query: {
                     alt: 'json',
                     key: 'AIzaSyCzEW7JUJdSql0-2V4tHUb6laYm4iAE_dM',
                 },
-                json: {
+                body: {
                     browseId: 'FEmusic_analytics_charts_home',
                     context: {
                         capabilities: {},

package/lib/routes/youtube/community.ts

@@ -6,7 +6,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseRelativeDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
-import * as path from 'node:path';
+import path from 'node:path';
 import { isYouTubeChannelId } from './utils';
 
 export const route: Route = {
@@ -14,14 +14,6 @@ export const route: Route = {
     categories: ['social-media'],
     example: '/youtube/community/@JFlaMusic',
     parameters: { handle: 'YouTube handles or channel id' },
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
     name: 'Community',
     maintainers: ['TonyRL'],
     handler,
@@ -40,7 +32,7 @@ async function handler(ctx) {
     const ytInitialData = JSON.parse(
         $('script')
             .text()
-            .match(/ytInitialData = ({.*?});/)[1]
+            .match(/ytInitialData = ({.*?});/)?.[1] ?? '{}'
     );
 
     const channelMetadata = ytInitialData.metadata.channelMetadataRenderer;