rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa04a11

package/lib/routes/oschina/column.ts

@@ -0,0 +1,246 @@
+import path from 'node:path';
+
+import { type CheerioAPI, type Cheerio, type Element, load } from 'cheerio';
+import { type Context } from 'hono';
+
+import { type DataItem, type Route, type Data, ViewType } from '@/types';
+
+import { art } from '@/utils/render';
+import cache from '@/utils/cache';
+import { getCurrentPath } from '@/utils/helpers';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+const __dirname = getCurrentPath(import.meta.url);
+
+export const handler = async (ctx: Context): Promise<Data> => {
+    const { id } = ctx.req.param();
+    const limit: number = Number.parseInt(ctx.req.query('limit') ?? '10', 10);
+
+    const baseUrl: string = 'https://www.oschina.net';
+    const userHostRegex: string = String.raw`https://my\.oschina\.net`;
+    const targetUrl: string = new URL(`news/column?columnId=${id}`, baseUrl).href;
+
+    const response = await ofetch(targetUrl);
+    const $: CheerioAPI = load(response);
+    const language: string = $('html').attr('lang') ?? 'zh-CN';
+
+    let items: DataItem[] = [];
+
+    items = $('div.news-item')
+        .slice(0, limit)
+        .toArray()
+        .map((el): Element => {
+            const $el: Cheerio<Element> = $(el);
+
+            const title: string = $el.find('div.title').text();
+            const description: string = art(path.join(__dirname, 'templates/description.art'), {
+                intro: $el.find('div.description p.line-clamp').text(),
+            });
+            const pubDateStr: string | undefined = $el.find('inddiv.item').contents().last().text().trim();
+            const linkUrl: string | undefined = $el.attr('data-url');
+            const authorEls: Element[] = $el.find('inddiv.item a').toArray();
+            const authors: DataItem['author'] = authorEls.map((authorEl) => {
+                const $authorEl: Cheerio<Element> = $(authorEl);
+
+                return {
+                    name: $authorEl.text(),
+                    url: $authorEl.attr('href'),
+                };
+            });
+            const image: string | undefined = $el.find('img').attr('src');
+            const upDatedStr: string | undefined = pubDateStr;
+
+            const processedItem: DataItem = {
+                title,
+                description,
+                pubDate: pubDateStr ? timezone(parseDate(pubDateStr), +8) : undefined,
+                link: linkUrl,
+                author: authors,
+                content: {
+                    html: description,
+                    text: description,
+                },
+                image,
+                banner: image,
+                updated: upDatedStr ? timezone(parseDate(upDatedStr), +8) : undefined,
+                language,
+            };
+
+            return processedItem;
+        });
+
+    items = (
+        await Promise.all(
+            items.map((item) => {
+                if (!item.link) {
+                    return item;
+                }
+
+                return cache.tryGet(item.link, async (): Promise<DataItem> => {
+                    const detailResponse = await ofetch(item.link);
+                    const $$: CheerioAPI = load(detailResponse);
+
+                    $$('.ad-wrap').remove();
+
+                    const title: string = $$('h1.article-box__title').text();
+                    const description: string = art(path.join(__dirname, 'templates/description.art'), {
+                        description: $$('div.content').html(),
+                    });
+                    const pubDateEl: Element = $$('div.article-box__meta div.item-list div.item')
+                        .toArray()
+                        .find((i) => /\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}/.test($$(i).text()));
+                    const pubDateStr: string | undefined = pubDateEl ? $$(pubDateEl).text() : undefined;
+                    const linkUrl: string | undefined = $$('val[data-name="shareUrl"]').attr('data-value');
+                    const categoryEls: Element[] = [...$$('div.breadcrumb-box a.item').toArray().slice(0, -1), ...$$('div.article-box__meta div.item-list div.item span.label').toArray(), ...$$('div.tags-box a.tag-item').toArray()];
+                    const categories: string[] = [...new Set(categoryEls.map((el) => $$(el).text()).filter(Boolean))];
+                    const authorEls: Element[] = $$('div.article-box__meta div.item-list div.item a')
+                        .toArray()
+                        .filter((i) => ($$(i).attr('href') ? new RegExp(`^${userHostRegex}/u/\\d+$`).test($$(i).attr('href') as string) : false));
+                    const authors: DataItem['author'] = authorEls.map((authorEl) => {
+                        const $authorEl: Cheerio<Element> = $$(authorEl);
+
+                        return {
+                            name: $authorEl.text(),
+                            url: $authorEl.attr('href'),
+                        };
+                    });
+                    const guid: string = `oschina-${$$('val[data-name="objId"]').attr('data-value')}`;
+                    const image: string | undefined = $$('val[data-name="sharePic"]').attr('data-value');
+                    const upDatedStr: string | undefined = $$('meta[property="bytedance:updated_time"]').attr('content') || pubDateStr;
+
+                    let processedItem: DataItem = {
+                        title,
+                        description,
+                        pubDate: pubDateStr ? timezone(parseDate(pubDateStr), +8) : item.pubDate,
+                        link: linkUrl ? new URL(linkUrl, baseUrl).href : item.link,
+                        category: categories,
+                        author: authors,
+                        guid,
+                        id: guid,
+                        content: {
+                            html: description,
+                            text: description,
+                        },
+                        image,
+                        banner: image,
+                        updated: upDatedStr ? parseDate(upDatedStr) : item.updated,
+                        language,
+                    };
+
+                    const extraLinkEls: Element[] = $$('div.related-links-box ul.link-list li a').toArray();
+                    const extraLinks = extraLinkEls
+                        .map((extraLinkEl) => {
+                            const $$extraLinkEl: Cheerio<Element> = $$(extraLinkEl);
+
+                            return {
+                                url: $$extraLinkEl.attr('href'),
+                                type: 'related',
+                                content_html: $$extraLinkEl.parent().html(),
+                            };
+                        })
+                        .filter((_): _ is { url: string; type: string; content_html: string } => true);
+
+                    if (extraLinks) {
+                        processedItem = {
+                            ...processedItem,
+                            _extra: {
+                                links: extraLinks,
+                            },
+                        };
+                    }
+
+                    return {
+                        ...item,
+                        ...processedItem,
+                    };
+                });
+            })
+        )
+    ).filter((_): _ is DataItem => true);
+
+    const author: string | undefined = $('a.logo').attr('title');
+
+    return {
+        title: `${author} - ${$('div#tabDropdownListOpen a.selected').text()}`,
+        description: $('meta[name="description"]').attr('content'),
+        link: targetUrl,
+        item: items,
+        allowEmpty: true,
+        image: $('a.logo img').attr('src'),
+        author,
+        language,
+        id: $('val[data-name="weixinShareUrl"]').attr('data-value'),
+    };
+};
+
+export const route: Route = {
+    path: '/column/:id',
+    name: '专栏',
+    url: 'www.oschina.net',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/oschina/column/14',
+    parameters: {
+        id: '专栏 id，可在对应专栏页 URL 中找到',
+    },
+    description: `:::tip
+若订阅 [开源安全专栏](https://www.oschina.net/news/column?columnId=14)，网址为 \`https://www.oschina.net/news/column?columnId=14\`，请截取 \`https://www.oschina.net/news/column?columnId=\` 到末尾的部分 \`14\` 作为 \`id\` 参数填入，此时目标路由为 [\`/oschina/column/14\`](https://rsshub.app/oschina/column/14)。
+
+:::
+
+<details>
+  <summary>更多专栏</summary>
+
+| 名称            | ID  |
+| --------------- | --- |
+| 古典主义 Debian | 4   |
+| 自由&开源       | 5   |
+| 溯源            | 6   |
+| 开源先懂协议    | 7   |
+| 开源变局        | 8   |
+| 创造者说        | 9   |
+| 精英主义 BSD    | 10  |
+| 苹果有开源      | 11  |
+| 开源访谈        | 12  |
+| 抱团找组织      | 13  |
+| 开源安全        | 14  |
+| OSPO            | 15  |
+| 创业小辑        | 16  |
+| 星推荐          | 17  |
+| 单口开源        | 18  |
+| 编辑部观察直播  | 19  |
+| 开源商业化      | 20  |
+| ChatGPT 专题    | 21  |
+| 开源新思        | 24  |
+| 开源日报        | 25  |
+| 大模型思辨      | 26  |
+| 家里有个程序员  | 27  |
+| 开源漫谈        | 23  |
+
+</details>
+`,
+    categories: ['programming'],
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['www.oschina.net'],
+            target: (_, url) => {
+                const urlObj: URL = new URL(url);
+                const id: string | undefined = urlObj.searchParams.get('id') ?? undefined;
+
+                return `/oschina/column/${id}`;
+            },
+        },
+    ],
+    view: ViewType.Articles,
+};

package/lib/routes/oschina/namespace.ts

@@ -4,4 +4,5 @@ export const namespace: Namespace = {
     name: '开源中国',
     url: 'oschina.net',
     lang: 'zh-CN',
+    description: 'OSCHINA',
 };

package/lib/routes/oschina/news.ts

@@ -1,10 +1,11 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { parseDate, parseRelativeDate } from '@/utils/parse-date';
 import timezone from '@/utils/timezone';
 import { load } from 'cheerio';
 import { fetchArticle } from '@/utils/wechat-mp';
+import { config } from '@/config';
 
 const configs = {
     all: {
@@ -73,17 +74,34 @@ export const route: Route = {
   [osc_pl]: https://www.oschina.net/news/programming "开源中国 - 编程语言资讯"`,
 };
 
+const getCookie = () =>
+    cache.tryGet(
+        'oschina:cookie',
+        async () => {
+            const res = await ofetch.raw('https://www.oschina.net/news');
+            const cookie = res.headers
+                .getSetCookie()
+                .map((i) => i.split(';')[0])
+                .join('; ');
+            return cookie;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
 async function handler(ctx) {
     const category = ctx.req.param('category') ?? 'all';
     const config = configs[category];
 
-    const res = await got(config.ajaxUrl, {
+    const cookie = await getCookie();
+    const res = await ofetch(config.ajaxUrl, {
         headers: {
             Referer: config.link,
             'X-Requested-With': 'XMLHttpRequest',
+            Cookie: cookie,
         },
     });
-    const $ = load(res.data);
+    const $ = load(res);
 
     $('.ad-wrap').remove();
 
@@ -104,23 +122,24 @@ async function handler(ctx) {
         list.map((item) =>
             cache.tryGet(item.link, async () => {
                 if (/^https?:\/\/(my|www)\.oschina.net\/.*$/.test(item.link)) {
-                    const detail = await got(item.link, {
+                    const detail = await ofetch(item.link, {
                         headers: {
                             Referer: config.link,
+                            Cookie: cookie,
                         },
                     });
-                    const content = load(detail.data);
+                    const content = load(detail);
                     content('.ad-wrap').remove();
 
                     item.description = content('.article-detail').html();
                     item.author = content('.article-box__meta .item').first().text();
                 } else if (/^https?:\/\/gitee\.com\/.*$/.test(item.link)) {
-                    const detail = await got(item.link, {
+                    const detail = await ofetch(item.link, {
                         headers: {
                             Referer: config.link,
                         },
                     });
-                    const content = load(detail.data);
+                    const content = load(detail);
 
                     item.description = content('.file_content').html();
                 } else if (/^https?:\/\/osc\.cool\/.*$/.test(item.link)) {

package/lib/routes/oschina/templates/description.art

@@ -0,0 +1,7 @@
+{{ if intro }}
+  <blockquote>{{ intro }}</blockquote>
+{{ /if }}
+
+{{ if description }}
+  {{@ description }}
+{{ /if }}

package/lib/routes/oschina/user.ts

@@ -69,7 +69,7 @@ async function handler(ctx) {
     return {
         title: author + '的博客',
         description: $('.user-text .user-signature').text(),
-        link: `https://my.oschina.net/${id ?? uid}`,
+        link: `https://my.oschina.net/u/${id ?? uid}`,
         item: resultItem,
     };
 }

package/lib/routes/ouc/jwgl.ts

@@ -26,9 +26,9 @@ export const route: Route = {
     maintainers: ['3401797899'],
     handler,
     url: 'jwgl.ouc.edu.cn/cas/login.action',
-    description: `:::warning
+    description: `::: warning
   由于选课通知仅允许校园网访问，需自行部署。
-  :::`,
+:::`,
 };
 
 async function handler() {

package/lib/routes/p-articles/namespace.ts

@@ -4,7 +4,7 @@ export const namespace: Namespace = {
     name: '虚词',
     url: 'p-articles.com',
     description: `
-:::tip
+::: tip
 p-articles provides some official RSS feeds:
 
 -   section: \`https://p-articles.com/section/:section\`

package/lib/routes/pacilution/latest.ts

@@ -0,0 +1,102 @@
+import { DataItem, Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+import iconv from 'iconv-lite';
+
+const BASE_URL = 'http://www.pacilution.com/';
+
+const handler: Route['handler'] = async () => {
+    // Fetch the target page
+    const response = await got({
+        method: 'get',
+        url: BASE_URL,
+        responseType: 'buffer',
+    });
+    const $ = load(iconv.decode(response.data, 'gb2312'));
+
+    // Select all list items containing target information
+    const ITEM_SELECTOR = 'ul[class*="ullbxwnew"] > li';
+    const listItems = $(ITEM_SELECTOR);
+
+    // Map through each list item to extract details
+    const contentLinkList = listItems.toArray().map((element) => {
+        const title = $(element).find('a').text();
+        const relativeHref = $(element).find('a').attr('href') || '';
+        const link = `${BASE_URL}${relativeHref}`;
+
+        return {
+            title,
+            link,
+        };
+    });
+
+    return {
+        title: '普世社会科学研究网最新文章',
+        description: '普世社会科学研究网首页上不同板块的最新文章汇总集合',
+        link: BASE_URL,
+        image: 'http://www.pacilution.com/img/top_banner.jpg',
+        item: (
+            await Promise.all(
+                contentLinkList.map((item) =>
+                    cache.tryGet(item.link, async () => {
+                        try {
+                            const CONTENT_SELECTOR = '#MyContent';
+                            const DATE_SELECTOR = 'td[class*="con_info"] > span';
+                            const response = await got({
+                                method: 'get',
+                                url: item.link,
+                                responseType: 'buffer',
+                            });
+                            const targetPage = load(iconv.decode(response.data, 'gb2312'));
+                            const content = targetPage(CONTENT_SELECTOR).html() || '';
+                            const date = parseDate(targetPage(DATE_SELECTOR).text().trim().replaceAll('日', '')).toISOString();
+                            return {
+                                title: item.title,
+                                pubDate: date,
+                                link: item.link,
+                                description: content,
+                                category: ['journal'],
+                                guid: item.link,
+                                id: item.link,
+                                image: 'http://www.pacilution.com/img/top_banner.jpg',
+                                content,
+                                updated: date,
+                                language: 'zh-cn',
+                            };
+                        } catch {
+                            return null as unknown as DataItem;
+                        }
+                    })
+                )
+            )
+        ).filter((item) => item !== null) as DataItem[],
+        allowEmpty: true,
+        language: 'zh-cn',
+        feedLink: 'https://rsshub.app/pacilution/latest',
+        id: 'https://rsshub.app/pacilution/latest',
+    };
+};
+
+export const route: Route = {
+    path: '/latest',
+    name: '最新文章',
+    maintainers: ['PrinOrange'],
+    handler,
+    categories: ['journal'],
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    example: '/pacilution/latest',
+    radar: [
+        {
+            source: ['www.pacilution.com'],
+        },
+    ],
+};

package/lib/routes/pacilution/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '普世社会科学研究所',
+    url: 'www.pacilution.com',
+};

package/lib/routes/panewslab/author.ts

@@ -6,7 +6,7 @@ import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: ['/author/:id', '/column/:id'],
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/panewslab/author/166',
     parameters: { id: '专栏 id，可在地址栏 URL 中找到' },
     features: {

package/lib/routes/panewslab/news.ts

@@ -4,7 +4,7 @@ import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/news',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/panewslab/news',
     radar: [
         {

package/lib/routes/panewslab/profundity.ts

@@ -21,7 +21,7 @@ const categories = {
 
 export const route: Route = {
     path: '/profundity/:category?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/panewslab/profundity',
     parameters: { category: '分类，见下表，默认为精选' },
     radar: [

package/lib/routes/panewslab/topic.ts

@@ -6,7 +6,7 @@ import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/topic/:id',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/panewslab/topic/1629365774078402',
     parameters: { id: '专题 id，可在地址栏 URL 中找到' },
     features: {

package/lib/routes/papers/index.ts

@@ -17,7 +17,7 @@ export const handler = async (ctx) => {
 
     const rootUrl = 'https://papers.cool';
     const currentUrl = new URL(category, rootUrl).href;
-    const feedUrl = new URL(`${category}/feed`, rootUrl).href;
+    const feedUrl = new URL(`arxiv/${category}/feed`, rootUrl).href;
 
     const site = category.split(/\//)[0];
     const apiKimiUrl = new URL(`${site}/kimi?paper=`, rootUrl).href;
@@ -34,10 +34,12 @@ export const handler = async (ctx) => {
         const kimiUrl = new URL(id, apiKimiUrl).href;
         const pdfUrl = Object.hasOwn(pdfUrlGenerators, site) ? pdfUrlGenerators[site](id) : undefined;
 
+        const authorString = item.author;
         const description = art(path.join(__dirname, 'templates/description.art'), {
             pdfUrl,
             siteUrl: item.link,
             kimiUrl,
+            authorString,
             summary: item.summary,
         });
 
@@ -47,7 +49,7 @@ export const handler = async (ctx) => {
             pubDate: parseDate(item.pubDate ?? ''),
             link: item.link,
             category: item.categories,
-            author: item.creator,
+            author: authorString,
             doi: `${site}${id}`,
             guid,
             id: guid,
@@ -74,16 +76,16 @@ export const handler = async (ctx) => {
 };
 
 export const route: Route = {
-    path: '/:category{.+}?',
+    path: '/arxiv/:category{.+}?',
     name: 'Topic',
     url: 'papers.cool',
-    maintainers: ['nczitzk'],
+    maintainers: ['nczitzk', 'Muyun99'],
     handler,
     example: '/papers/arxiv/cs.AI',
     parameters: { category: 'Category, arXiv Artificial Intelligence (cs.AI) by default' },
-    description: `:::tip
-  If you subscribe to [arXiv Artificial Intelligence (cs.AI)](https://papers.cool/arxiv/cs.AI)，where the URL is \`https://papers.cool/arxiv/cs.AI\`, extract the part \`https://papers.cool/\` to the end, and use it as the parameter to fill in. Therefore, the route will be [\`/papers/arxiv/cs.AI\`](https://rsshub.app/papers/arxiv/cs.AI).
-  :::
+    description: `::: tip
+  If you subscribe to [arXiv Artificial Intelligence (cs.AI)](https://papers.cool/arxiv/cs.AI), where the URL is \`https://papers.cool/arxiv/cs.AI\`, extract the part \`https://papers.cool/\` to the end, and use it as the parameter to fill in. Therefore, the route will be [\`/papers/arxiv/cs.AI\`](https://rsshub.app/papers/arxiv/cs.AI).
+:::
 
   | Category                                              | id          |
   | ----------------------------------------------------- | ----------- |
@@ -91,7 +93,7 @@ export const route: Route = {
   | arXiv Computation and Language (cs.CL)                | arxiv/cs.CL |
   | arXiv Computer Vision and Pattern Recognition (cs.CV) | arxiv/cs.CV |
   | arXiv Machine Learning (cs.LG)                        | arxiv/cs.LG |
-  | arXiv Robotics (cs.RO)                        | arxiv/cs.RO |
+  | arXiv Robotics (cs.RO)                                | arxiv/cs.RO |
   `,
     categories: ['journal'],