rsshub 1.0.0-master.ff3a9ce → 1.0.0-master.ff3ea5a

package/lib/routes/sankei/news.ts

@@ -0,0 +1,68 @@
+import { Route, Data, DataItem } from '@/types';
+
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import { Context } from 'hono';
+
+export const route: Route = {
+    path: '/news/:category',
+    categories: ['traditional-media'],
+    example: '/sankei/news/flash',
+    parameters: { category: 'Category name (as it will appear in URLs). For example, for "Breaking News" https://www.sankei.com/flash/, the category name would be "flash".' },
+    radar: [
+        {
+            source: ['www.sankei.com/:category'],
+            target: '/news/:category',
+        },
+    ],
+    name: 'News',
+    maintainers: ['yuikisaito'],
+    handler,
+};
+
+async function handler(ctx: Context): Promise<Data> {
+    const baseUrl = 'https://www.sankei.com';
+    const { category } = ctx.req.param();
+    const url = `${baseUrl}/${category}/`;
+
+    const response = await got(url);
+    const $ = load(response.body);
+    const categoryName = $('header.story-card-header:first-of-type > h1').text();
+    const listSelector = $('div.story-card-feed.grid.hide_on_mobile > div > article');
+
+    const items: DataItem[] = await Promise.all(
+        listSelector.toArray().map((el) => {
+            const item = $(el);
+            item.find('p a').remove();
+
+            const title = item.find('div.story-card-flex > h2.headline > a').text();
+            const link = baseUrl + (item.find('div.story-card-flex > h2.headline > a').attr('href') || '');
+            const pubDate = parseDate(item.find('div.story-card-flex > div > time').attr('datetime') || '');
+
+            return cache.tryGet(link, async () => {
+                const detail = await got(link);
+                const $ = load(detail.body);
+                $('.inline-gptAd, .figure_image_sizer').remove();
+                const articleHTML = $('div.article-body').html() || '';
+
+                return {
+                    title,
+                    link,
+                    pubDate,
+                    description: articleHTML,
+                };
+            });
+        })
+    );
+
+    return {
+        title: '産経ニュース - ' + categoryName,
+        description: $('meta[name="description"]').attr('content'),
+        link: url,
+        image: $('meta[property="og:image"]').attr('content'),
+        language: 'ja',
+        item: items,
+    };
+}

package/lib/routes/sankei/topics.ts

@@ -0,0 +1,71 @@
+import { Route, Data, DataItem } from '@/types';
+
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import { Context } from 'hono';
+
+export const route: Route = {
+    path: ['/topics/:topic'],
+    categories: ['traditional-media'],
+    example: '/sankei/topics/etc_100',
+    parameters: {
+        topic: 'Topic name (format included in URL). For example, for "Expo 2025 Osaka, Kansai, Japan Special Feature" https://www.sankei.com/tag/topic/etc_100, the value would be etc_100.',
+    },
+    radar: [
+        {
+            source: ['www.sankei.com/tag/topic/:topic'],
+            target: '/topics/:topic',
+        },
+    ],
+    name: 'Topic',
+    maintainers: ['yuikisaito'],
+    handler,
+};
+
+async function handler(ctx: Context): Promise<Data> {
+    const baseUrl = 'https://www.sankei.com';
+    const { topic } = ctx.req.param();
+    const url = `${baseUrl}/tag/topic/${topic}/`;
+
+    const response = await got(url);
+    const $ = load(response.body);
+    const categoryName = $('li.breadcrumb-list-item:nth-of-type(3) > a').text();
+
+    const listSelector = $('div.story-card-feed.grid.hide_on_mobile > div > article');
+
+    const items: DataItem[] = await Promise.all(
+        listSelector.toArray().map((el) => {
+            const item = $(el);
+            item.find('p a').remove();
+
+            const title = item.find('div.story-card-flex > h3.headline > a').text();
+            const link = baseUrl + (item.find('div.story-card-flex > h3.headline > a').attr('href') || '');
+            const pubDate = parseDate(item.find('div.story-card-flex > div > time').attr('datetime') || '');
+
+            return cache.tryGet(link, async () => {
+                const detail = await got(link);
+                const $ = load(detail.body);
+                $('.inline-gptAd, .figure_image_sizer').remove();
+                const articleHTML = $('div.article-body').html() || '';
+
+                return {
+                    title,
+                    link,
+                    pubDate,
+                    description: articleHTML,
+                };
+            });
+        })
+    );
+
+    return {
+        title: '産経ニュース - ' + categoryName,
+        description: $('meta[name="description"]').attr('content'),
+        link: url,
+        image: $('meta[property="og:image"]').attr('content'),
+        language: 'ja',
+        item: items,
+    };
+}

package/lib/routes/sciencenet/user.ts

@@ -54,7 +54,7 @@ async function handler(ctx) {
     let items = $('item')
         .slice(-limit)
         .toArray()
-        .reverse()
+        .toReversed()
         .map((item) => {
             item = $(item);
 

package/lib/routes/scoop/apps.ts

@@ -0,0 +1,188 @@
+import { type Data, type DataItem, type Route, ViewType } from '@/types';
+
+import { art } from '@/utils/render';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+
+import { type CheerioAPI, load } from 'cheerio';
+import { type Context } from 'hono';
+import path from 'node:path';
+
+const orderbys = (desc: string) => {
+    const base = {
+        0: 'search.score() desc, Metadata/OfficialRepositoryNumber desc, NameSortable asc',
+        1: 'NameSortable asc, Metadata/OfficialRepositoryNumber desc, Metadata/RepositoryStars desc, Metadata/Committed desc',
+        2: 'Metadata/Committed desc, Metadata/OfficialRepositoryNumber desc, Metadata/RepositoryStars desc',
+    };
+
+    if (desc === '1') {
+        return base;
+    }
+
+    const inverted = {};
+    for (const key in base) {
+        const orderStr = base[key];
+        inverted[key] = orderStr.replaceAll(/\b(desc|asc)\b/gi, (match) => (match.toLowerCase() === 'desc' ? 'asc' : 'desc'));
+    }
+    return inverted;
+};
+
+const filters = {
+    o: 'Metadata/OfficialRepositoryNumber eq 1', // offical buckets only
+    dm: 'Metadata/DuplicateOf eq null', // distinct manifests only
+};
+
+export const handler = async (ctx: Context): Promise<Data> => {
+    const { query = 's=2&d=1&n=true&dm=true&o=true' } = ctx.req.param();
+    const limit: number = Number.parseInt(ctx.req.query('limit') ?? '50', 10);
+
+    const baseUrl: string = 'https://scoop.sh';
+    const apiBaseUrl: string = 'https://scoopsearch.search.windows.net';
+    const targetUrl: string = new URL(`/#/apps?${query}`, baseUrl).href;
+    const apiUrl: string = new URL('indexes/apps/docs/search', apiBaseUrl).href;
+
+    const targetResponse = await ofetch(targetUrl);
+    const $: CheerioAPI = load(targetResponse);
+    const language = $('html').attr('lang') ?? 'en';
+
+    const scriptRegExp: RegExp = /<script type="module" crossorigin src="(.*?)"><\/script>/;
+    const scriptUrl: string = scriptRegExp.test(targetResponse) ? new URL(targetResponse.match(scriptRegExp)?.[1], baseUrl).href : '';
+
+    if (!scriptUrl) {
+        throw new Error('JavaScript file not found.');
+    }
+
+    const scriptResponse = await ofetch(scriptUrl, {
+        parseResponse: (txt) => txt,
+    });
+
+    const key: string = scriptResponse.match(/VITE_APP_AZURESEARCH_KEY:"(.*?)"/)?.[1];
+
+    if (!key) {
+        throw new Error('Key not found.');
+    }
+
+    const isOffcial: boolean = !query.includes('o=false');
+    const isDistinct: boolean = !query.includes('dm=false');
+    const sort: string = query.match(/s=(\d+)/)?.[1] ?? '2';
+    const desc: string = query.match(/d=(\d+)/)?.[1] ?? '1';
+
+    const response = await ofetch(apiUrl, {
+        method: 'post',
+        query: {
+            'api-version': '2020-06-30',
+        },
+        headers: {
+            'api-key': key,
+            origin: baseUrl,
+            referer: baseUrl,
+        },
+        body: {
+            count: true,
+            search: '',
+            searchMode: 'all',
+            filter: [isOffcial ? filters.o : undefined, isDistinct ? filters.dm : undefined].filter(Boolean).join(' and '),
+            orderby: orderbys(desc)[sort],
+            skip: 0,
+            top: limit,
+            select: 'Id,Name,NamePartial,NameSuffix,Description,Notes,Homepage,License,Version,Metadata/Repository,Metadata/FilePath,Metadata/OfficialRepository,Metadata/RepositoryStars,Metadata/Committed,Metadata/Sha',
+            highlight: 'Name,NamePartial,NameSuffix,Description,Version,License,Metadata/Repository',
+            highlightPreTag: '<mark>',
+            highlightPostTag: '</mark>',
+        },
+    });
+
+    let items: DataItem[] = [];
+
+    items = response.value.slice(0, limit).map((item): DataItem => {
+        const repositorySplits: string[] = item.Metadata.Repository.split(/\//);
+        const repositoryName: string = repositorySplits.slice(-2).join('/');
+        const title: string = `${item.Name} ${item.Version} in ${repositoryName}`;
+        const description: string | undefined = art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+        const pubDate: number | string = item.Metadata.Committed;
+        const linkUrl: string | undefined = item.Homepage;
+        const authors: DataItem['author'] = [
+            {
+                name: repositoryName,
+                url: item.Metadata.Repository,
+                avatar: undefined,
+            },
+        ];
+        const guid: string = `scoop-${item.Name}-${item.Version}-${item.Metadata.Sha}`;
+        const updated: number | string = pubDate;
+
+        const processedItem: DataItem = {
+            title,
+            description,
+            pubDate: pubDate ? parseDate(pubDate) : undefined,
+            link: linkUrl,
+            author: authors,
+            guid,
+            id: guid,
+            content: {
+                html: description,
+                text: description,
+            },
+            updated: updated ? parseDate(updated) : undefined,
+            language,
+        };
+
+        return processedItem;
+    });
+
+    const author: string = 'Scoop';
+
+    return {
+        title: `${author} - Apps`,
+        description: undefined,
+        link: targetUrl,
+        item: items,
+        allowEmpty: true,
+        author,
+        language,
+        id: targetUrl,
+    };
+};
+
+export const route: Route = {
+    path: '/apps/:query?',
+    name: 'Apps',
+    url: 'scoop.sh',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/scoop/apps',
+    parameters: {
+        query: {
+            description: 'Query, `s=2&d=1&n=true&dm=true&o=true` by default',
+        },
+    },
+    description: `:::tip
+To subscribe to [Apps](https://scoop.sh/#/apps?s=2&d=1&n=true&dm=true&o=true), where the source URL is \`https://scoop.sh/#/apps?s=2&d=1&n=true&dm=true&o=true\`, extract the certain parts from this URL to be used as parameters, resulting in the route as [\`/scoop/apps/s=2&d=1&n=true&dm=true&o=true\`](https://rsshub.app/scoop/apps/s=2&d=1&n=true&dm=true&o=true).
+
+:::
+`,
+    categories: ['program-update'],
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['scoop.sh/#/apps', 'scoop.sh'],
+            target: (_, url) => {
+                const urlObj: URL = new URL(url);
+                const query: string | undefined = urlObj.searchParams.toString() ?? undefined;
+
+                return `/scoop/apps${query ? `/${query}` : ''}`;
+            },
+        },
+    ],
+    view: ViewType.Notifications,
+};

package/lib/routes/scoop/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Scoop',
+    url: 'scoop.sh',
+    categories: ['program-update'],
+    description: '',
+    lang: 'en',
+};

package/lib/routes/scoop/templates/description.art

@@ -0,0 +1,56 @@
+{{ if item }}
+  <table>
+    <tbody>
+    {{ if item.Name }}
+      <tr>
+        <th>Name</th>
+        <td>{{ item.Name }}</td>
+      </tr>
+    {{ /if }}
+    {{ if item.Repository }}
+      <tr>
+        <th>Repository</th>
+        <td>
+          <a href="{{ item.Metadata.Repository }}">{{ item.Metadata.Repository.split(/\//).slice(-2).join('/') }}</a>
+        </td>
+      </tr>
+    {{ /if }}
+    {{ if item.Committed }}
+      <tr>
+        <th>Committed</th>
+        <td><a href="{{ item.Metadata.Repository }}/commit/{{ item.Metadata.Sha }}">{{ item.Metadata.Committed }}</a></td>
+      </tr>
+    {{ /if }}
+    {{ if item.Version }}
+      <tr>
+        <th>Version</th>
+        <td><a href="{{ item.Metadata.Repository }}/blob/{{ item.Metadata.FilePath }}">v{{ item.Version }}</a></td>
+      </tr>
+    {{ /if }}
+    {{ if item.Description }}
+      <tr>
+        <th>Description</th>
+        <td>{{ item.Description }}</td>
+      </tr>
+    {{ /if }}
+    {{ if item.Homepage }}
+      </tr>
+        <th>Homepage</th>
+        <td><a href="{{ item.Homepage }}">{{ item.Homepage }}</a></td>
+      </tr>
+    {{ /if }}
+    {{ if item.License }}
+      <tr>
+        <th>License</th>
+        <td>{{ item.License }}</td>
+      </tr>
+    {{ /if }}
+    {{ if item.Note }}
+      <tr>
+        <th>Note</th>
+        <td>{{ item.Note }}</td>
+      </tr>
+    {{ /if }}
+    </tbody>
+  </table>
+{{ /if }}

package/lib/routes/scpta/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '四川省人力资源和社会保障厅人事考试专栏',
+    url: 'www.scpta.com.cn',
+    categories: ['government'],
+    lang: 'zh-CN',
+};

package/lib/routes/scpta/news.ts

@@ -0,0 +1,101 @@
+import { Route } from '@/types';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import cache from '@/utils/cache';
+
+export const route: Route = {
+    path: '/news/:category',
+    categories: ['government'],
+    example: '/scpta/news/33',
+    parameters: {
+        category: {
+            description: '分类ID，默认为`33`(工作动态)',
+            default: '33',
+        },
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['www.scpta.com.cn/front/News/List'],
+            target: '/news',
+        },
+    ],
+    name: '通知公告',
+    maintainers: ['Yeye-0426'], // github ID
+    handler,
+    description: `| 分类                 | category_id |
+|----------------------|-------------|
+| 工作动态             | 33          |
+| 公务员考试           | 56          |
+| 专业技术人员资格考试 | 57          |
+| 事业单位考试         | 67          |
+| 其它                 | 72          |`,
+};
+
+async function handler(ctx) {
+    const category = ctx.req.param('category');
+    const baseUrl = 'https://www.scpta.com.cn';
+
+    // 构建请求URL
+    const url = `${baseUrl}/front/News/List/${category}`;
+
+    // 发送请求
+    const response = await got(url);
+
+    const $ = load(response.data);
+
+    // 解析搜索结果
+    const list = $('div.wrap-content li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: `${baseUrl}${item.find('a').attr('href')}`,
+                pubDate: parseDate(item.find('span').text().trim()),
+            };
+        });
+    // 获取公告详情
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                let description = '';
+                try {
+                    const contentResponse = await got(item.link);
+                    const content = load(contentResponse.data);
+
+                    // 提取公告正文
+                    description = content('div.wrap-content.news-content').html();
+                } catch {
+                    // 如果详情页获取失败，使用默认描述
+                    description = '公告内容获取失败';
+                }
+                item.description = description;
+                return item;
+            })
+        )
+    );
+
+    // 生成RSS输出
+    const categoryNames = {
+        '33': '工作动态',
+        '56': '公务员考试',
+        '57': '专业技术人员资格考试',
+        '67': '事业单位考试',
+        '72': '其它',
+    };
+
+    return {
+        title: `通知公告 - ${categoryNames[category] || '未知分类'}`,
+        link: url,
+        item: items,
+    };
+}

package/lib/routes/seekingalpha/index.ts

@@ -64,7 +64,7 @@ async function handler(ctx) {
             'filter[until]': 0,
             id: symbol.toLowerCase(),
             include: 'author,primaryTickers,secondaryTickers,sentiments',
-            'page[size]': ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : (category === 'news' ? 40 : 20),
+            'page[size]': ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : category === 'news' ? 40 : 20,
             'page[number]': 1,
         },
     });

package/lib/routes/sehuatang/index.ts

@@ -1,10 +1,10 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 import timezone from '@/utils/timezone';
-import logger from '@/utils/logger';
-import puppeteer from '@/utils/puppeteer';
+import { config } from '@/config';
 
 const host = 'https://www.sehuatang.net/';
 
@@ -42,10 +42,10 @@ export const route: Route = {
     path: ['/bt/:subforumid?', '/picture/:subforumid', '/:subforumid?/:type?', '/:subforumid?', ''],
     name: 'Forum',
     maintainers: ['qiwihui', 'junfengP', 'nczitzk'],
+    handler,
     features: {
-        requirePuppeteer: true,
+        nsfw: true,
     },
-    handler,
     description: `**原创 BT 电影**
 
 | 国产原创 | 亚洲无码原创 | 亚洲有码原创 | 高清中文字幕 | 三级写真 | VR 视频 | 素人有码 | 欧美无码 | 韩国主播 | 动漫原创 | 综合讨论 |
@@ -59,29 +59,34 @@ export const route: Route = {
 | yczp     | ztzp     | hrjp     | yzxa     | omxa     | ktdm     | ttxz     |`,
 };
 
+const getSafeId = () =>
+    cache.tryGet(
+        'sehuatang:safeid',
+        async () => {
+            const response = await ofetch(host);
+            const $ = load(response);
+            const safeId = $('script:contains("safeid")')
+                .text()
+                .match(/safeid\s*=\s*'(.+)';/)?.[1];
+            return safeId;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
 async function handler(ctx) {
     const subformName = ctx.req.param('subforumid') ?? 'gqzwzm';
     const subformId = subformName in forumIdMaps ? forumIdMaps[subformName] : subformName;
     const type = ctx.req.param('type');
     const typefilter = type ? `&filter=typeid&typeid=${type}` : '';
     const link = `${host}forum.php?mod=forumdisplay&orderby=dateline&fid=${subformId}${typefilter}`;
+    const headers = {
+        Cookie: `_safe=${await getSafeId()};`,
+    };
 
-    const browser = await puppeteer();
-    const page = await browser.newPage();
-    await page.setRequestInterception(true);
-    page.on('request', (request) => {
-        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
-    });
-    logger.http(`Requesting ${link}`);
-    await page.goto(link, {
-        waitUntil: 'domcontentloaded',
+    const response = await ofetch(link, {
+        headers,
     });
-    await page.waitForSelector('a.enter-btn', { visible: true });
-
-    await Promise.all([page.click('a.enter-btn'), page.waitForNavigation({ waitUntil: 'domcontentloaded' })]);
-    const response = await page.content();
-    await page.close();
-
     const $ = load(response);
 
     const list = $('#threadlisttableid tbody[id^=normalthread]')
@@ -101,20 +106,12 @@ async function handler(ctx) {
     const out = await Promise.all(
         list.map((info) =>
             cache.tryGet(info.link, async () => {
-                const page = await browser.newPage();
-                await page.setRequestInterception(true);
-                page.on('request', (request) => {
-                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                const response = await ofetch(info.link, {
+                    headers,
                 });
 
-                await page.goto(info.link, {
-                    // 指定页面等待载入的时间
-                    waitUntil: 'domcontentloaded',
-                });
-                const response = await page.content();
-                await page.close();
                 const $ = load(response);
-                const postMessage = $("td[id^='postmessage']").slice(0, 1);
+                const postMessage = $('div[id^="postmessage"], td[id^="postmessage"]').slice(0, 1);
                 const images = $(postMessage).find('img');
                 for (const image of images) {
                     const file = $(image).attr('file');
@@ -151,11 +148,12 @@ async function handler(ctx) {
                     info.enclosure_url = enclosureUrl;
                     info.enclosure_type = isMag ? 'application/x-bittorrent' : 'application/octet-stream';
                 }
+
                 return info;
             })
         )
     );
-    await browser.close();
+
     return {
         title: `色花堂 - ${$('#pt > div:nth-child(1) > a:last-child').text()}`,
         link,

package/lib/routes/setn/index.ts

@@ -2,6 +2,7 @@ import { Route } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
+import timezone from '@/utils/timezone';
 import { parseDate } from '@/utils/parse-date';
 
 const defaultRootUrl = 'https://www.setn.com';
@@ -118,11 +119,19 @@ async function handler(ctx) {
 
                 const content = load(detailResponse.data);
 
+                let head = {};
+                try {
+                    head = JSON.parse(content('script[type="application/ld+json"]').first().text());
+                } catch {
+                    head = {};
+                }
+
                 content('#gad_setn_innity_oop_1x1').remove();
 
-                item.author = content('meta[property="author"]').attr('content');
+                item.title = content('h1').text();
+                item.author = head?.author?.name || content('meta[name="author"]').attr('content');
                 item.category = [content('meta[property="article:section"]').attr('content'), ...content('meta[name="news_keywords"]').attr('content').split(',')];
-                item.pubDate = parseDate(content('meta[property="article:published_time"]').attr('content'));
+                item.pubDate = timezone(parseDate(content('meta[property="article:published_time"]').attr('content')), +8);
                 item.description = content('article, .content-p').html();
 
                 return item;