rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa1dcc5

package/lib/routes/kcna/news.ts

@@ -7,9 +7,11 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import asyncPool from 'tiny-async-pool';
 import { art } from '@/utils/render';
-import { parseJucheDate, fixDesc, fetchPhoto, fetchVideo } from './utils';
+import { fixDesc, fetchPhoto, fetchVideo } from './utils';
 import path from 'node:path';
 import sanitizeHtml from 'sanitize-html';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
 
 export const route: Route = {
     path: '/:lang/:category?',
@@ -66,12 +68,12 @@ async function handler(ctx) {
         .map((_, item) => {
             item = $(item);
             const dateElem = item.find('.publish-time');
-            const dateString = dateElem.text();
+            const dateString = dateElem.text().match(/\d+\.\d+\.\d+/);
             dateElem.remove();
             return {
                 title: item.text(),
                 link: rootUrl + item.attr('href'),
-                pubDate: parseJucheDate(dateString),
+                pubDate: timezone(parseDate(dateString[0]), +9),
             };
         })
         .get();
@@ -87,9 +89,9 @@ async function handler(ctx) {
             item.title = $('article-main-title').text() || item.title;
 
             const dateElem = $('.publish-time');
-            const dateString = dateElem.text();
+            const dateString = dateElem.text().match(/\d+\.\d+\.\d+/);
             dateElem.remove();
-            item.pubDate = parseJucheDate(dateString) || item.pubDate;
+            item.pubDate = dateString ? timezone(parseDate(dateString[0]), +9) : item.pubDate;
 
             const description = fixDesc($, $('.article-content-body .content-wrapper'));
 

package/lib/routes/keep/user.ts

@@ -8,7 +8,7 @@ import path from 'node:path';
 
 export const route: Route = {
     path: '/user/:id',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/keep/user/556b02c1ab59390afea671ea',
     parameters: { id: 'Keep 用户 id' },
     features: {

package/lib/routes/langchain/index.ts

@@ -0,0 +1,74 @@
+import { Route, DataItem } from '@/types';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import cache from '@/utils/cache';
+
+export const route: Route = {
+    path: '/blog',
+    categories: ['blog'],
+    example: '/langchain/blog',
+    radar: [
+        {
+            source: ['blog.langchain.dev/'],
+        },
+    ],
+    url: 'blog.langchain.dev/',
+    name: 'Blog',
+    maintainers: ['liyaozhong'],
+    handler,
+    description: 'LangChain Blog Posts',
+};
+
+async function handler() {
+    const rootUrl = 'https://blog.langchain.dev';
+    const currentUrl = rootUrl;
+
+    const response = await got(currentUrl);
+    const $ = load(response.data);
+
+    const items = await Promise.all(
+        $('.posts-feed .post-card')
+            .toArray()
+            .map((item) => {
+                const $item = $(item);
+                const $link = $item.find('.post-card__content-link').first();
+
+                const href = $link.attr('href');
+                const title = $item.find('.post-card__title').text().trim();
+                const excerpt = $item.find('.post-card__excerpt').text().trim();
+
+                if (!href || !title) {
+                    return null;
+                }
+
+                const link = new URL(href, rootUrl).href;
+
+                return {
+                    title,
+                    description: excerpt,
+                    link,
+                } as DataItem;
+            })
+            .filter((item): item is DataItem => item !== null)
+            .map((item) =>
+                cache.tryGet(item.link as string, async () => {
+                    try {
+                        const detailResponse = await got(item.link);
+                        const $detail = load(detailResponse.data);
+
+                        item.description = $detail('.article-content').html() || item.description;
+
+                        return item as DataItem;
+                    } catch {
+                        return item;
+                    }
+                })
+            )
+    );
+
+    return {
+        title: 'LangChain Blog',
+        link: rootUrl,
+        item: items.filter((item): item is DataItem => item !== null),
+    };
+}

package/lib/routes/langchain/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'LangChain Blog',
+    url: 'blog.langchain.dev',
+    lang: 'en',
+};

package/lib/routes/last-origin/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'LastOrigin',
+    url: 'www.last-origin.com',
+    lang: 'ja',
+};

package/lib/routes/last-origin/news.ts

@@ -0,0 +1,65 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+export const route: Route = {
+    path: '/news',
+    name: 'News',
+    url: 'www.last-origin.com',
+    maintainers: ['gudezhi'],
+    example: '/last-origin/news',
+    parameters: {},
+    categories: ['game'],
+    features: {
+        supportRadar: true,
+    },
+    radar: [
+        {
+            source: ['www.last-origin.com/news.html', 'www.last-origin.com'],
+            target: '/news',
+        },
+    ],
+    handler,
+    description: '',
+};
+
+async function handler() {
+    const baseUrl = 'https://www.last-origin.com/news.html';
+    const response = await ofetch(baseUrl);
+    const $ = load(response);
+
+    const list = $('.contents .news_wrap')
+        .toArray()
+        .map((item) => {
+            const title = $(item).find('.news_title').text().trim();
+            const link = new URL($(item).find('a').attr('href')!, baseUrl).href;
+            const date = $(item).find('time').text().trim();
+            const pubDate = timezone(parseDate(date), +9);
+            return {
+                title,
+                link,
+                pubDate,
+                description: '',
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await ofetch(item.link);
+                const $ = load(response);
+                item.description = $('.news_contents_editor').html() ?? '';
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: 'LastOrigin官网公告',
+        link: baseUrl,
+        item: items,
+    };
+}

package/lib/routes/linkresearcher/index.ts

@@ -1,76 +1,138 @@
-import { Route } from '@/types';
-import got from '@/utils/got';
-import qs from 'query-string';
+import { ViewType, type Data, type DataItem, type Route } from '@/types';
+import ofetch from '@/utils/ofetch';
 import { parseDate } from '@/utils/parse-date';
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+import crypto from 'crypto';
+import type { Context } from 'hono';
+import type { DetailResponse, SearchResultItem } from './types';
+import cache from '@/utils/cache';
+import { getCurrentPath } from '@/utils/helpers';
+import { art } from '@/utils/render';
+import path from 'node:path';
+
+const __dirname = getCurrentPath(import.meta.url);
+const templatePath = path.join(__dirname, 'templates/bilingual.art');
 
 const baseURL = 'https://www.linkresearcher.com';
+const apiURL = `${baseURL}/api`;
 
 export const route: Route = {
+    name: 'Articles',
     path: '/:params',
-    name: 'Unknown',
-    maintainers: ['yech1990'],
+    example: '/linkresearcher/category=theses&columns=Nature%20导读&subject=生物',
+    maintainers: ['y9c', 'KarasuShin'],
     handler,
+    view: ViewType.Articles,
+    categories: ['journal'],
+    parameters: {
+        params: {
+            description: 'search parameters, support `category`, `subject`, `columns`, `query`',
+        },
+    },
+    zh: {
+        name: '文章',
+    },
+    'zh-TW': {
+        name: '文章',
+    },
 };
 
-async function handler(ctx) {
-    // parse params
+async function handler(ctx: Context): Promise<Data> {
+    const categoryMap = { theses: '论文', information: '新闻', careers: '职业' } as const;
     const params = ctx.req.param('params');
-    const query = qs.parse(params);
-
-    const categoryMap = { theses: '论文', information: '新闻', careers: '职业' };
-    const category = query.category;
-    let title = categoryMap[category];
-
-    // get XSRF token from main page
-    const metaURL = `${baseURL}/${category}`;
-    const metaResponse = await got(metaURL);
-    const xsrfToken = metaResponse.headers['set-cookie'][0].split(';')[0].split('=')[1];
-
-    let data = { filters: { status: false } };
-    if (query.subject !== undefined && query.columns !== undefined) {
-        data = { filters: { status: true, subject: query.subject, columns: query.columns } };
-        title = `${title}「${query.subject} & ${query.columns}」`;
-    } else if (query.subject !== undefined && query.columns === undefined) {
-        data = { filters: { status: true, subject: query.subject } };
-        title = `${title}「${query.subject}」`;
-    } else if (query.subject === undefined && query.columns !== undefined) {
-        data = { filters: { status: true, columns: query.columns } };
-        title = `${title}「${query.columns}」`;
+    const filters = new URLSearchParams(params);
+
+    const subject = filters.get('subject');
+    const columns = filters.get('columns');
+    const query = filters.get('query') ?? '';
+    const category = filters.get('category') ?? ('theses' as keyof typeof categoryMap);
+
+    if (!(category in categoryMap)) {
+        throw new InvalidParameterError('Invalid category');
+    }
+    let title = categoryMap[category] as string;
+
+    const token = crypto.randomUUID();
+
+    const data: {
+        filters: {
+            status: boolean;
+            subject?: string;
+            columns?: string;
+        };
+    } = { filters: { status: true } };
+
+    if (subject) {
+        data.filters.subject = subject;
+        title = `${title}「${subject}」`;
     }
-    data.query = query.query;
+
+    if (columns) {
+        data.filters.columns = columns;
+        title = `${title}「${columns}」`;
+    }
+
     const dataURL = `${baseURL}/api/${category === 'careers' ? 'articles' : category}/search`;
-    const pageResponse = await got.post(dataURL, {
+    const pageResponse = await ofetch<{
+        hits: SearchResultItem[];
+    }>(dataURL, {
+        method: 'POST',
         headers: {
             'content-type': 'application/json; charset=UTF-8',
-            'x-xsrf-token': xsrfToken,
-            cookie: `XSRF-TOKEN=${xsrfToken}`,
+            'x-xsrf-token': token,
+            cookie: `XSRF-TOKEN=${token}`,
         },
-        searchParams: {
+        params: {
             from: 0,
             size: 20,
             type: category === 'careers' ? 'CAREER' : 'SEARCH',
         },
-        json: data,
+        body: {
+            ...data,
+            query,
+        },
     });
 
-    const list = pageResponse.data.hits;
+    const items = await Promise.all(
+        pageResponse.hits.map((item) => {
+            const link = `${baseURL}/${category}/${item.id}`;
+            return cache.tryGet(link, async () => {
+                const response = await ofetch<DetailResponse>(`${apiURL}/${category === 'theses' ? 'theses' : 'information'}/${item.id}`, {
+                    responseType: 'json',
+                });
+
+                const dataItem: DataItem = {
+                    title: response.title,
+                    pubDate: parseDate(response.onlineTime),
+                    link,
+                    image: response.cover,
+                };
+
+                dataItem.description =
+                    'zhTextList' in response && 'enTextList' in response
+                        ? art(templatePath, {
+                              zh: response.zhTextList,
+                              en: response.enTextList,
+                          })
+                        : response.content;
+
+                if ('paperList' in response) {
+                    const { doi, authors } = response.paperList[0];
+                    dataItem.doi = doi;
+                    dataItem.author = authors.map((author) => ({ name: author }));
+                }
 
-    const out = list.map((item) => ({
-        title: item.title,
-        description: item.content,
-        pubDate: parseDate(item.createdAt, 'x'),
-        link: `${metaURL}/${item.id}`,
-        guid: `${metaURL}/${item.id}`,
-        doi: item.identCode === undefined ? '' : item.identCode,
-        author: item.authors === undefined ? '' : item.authors.join(', '),
-    }));
+                return dataItem;
+            }) as unknown as DataItem;
+        })
+    );
 
     return {
         title: `领研 | ${title}`,
         description:
             '领研是链接华人学者的人才及成果平台。领研为国内外高校、科研机构及科技企业提供科研人才招聘服务，也是青年研究者的职业发展指导及线上培训平台；研究者还可将自己的研究论文上传至领研，与超过五十万华人学者分享工作的最新进展。',
-        image: 'https://www.linkresearcher.com/assets/images/logo-app.png',
+        image: `${baseURL}/assets/images/logo-app.png`,
         link: baseURL,
-        item: out,
+        item: items,
     };
 }

package/lib/routes/linkresearcher/namespace.ts

@@ -2,6 +2,12 @@ import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
     name: 'Link Research',
-    url: 'linkresearcher',
-    lang: 'en',
+    url: 'www.linkresearcher.com',
+    lang: 'zh-CN',
+    zh: {
+        name: '领研',
+    },
+    'zh-TW': {
+        name: '領研',
+    },
 };

package/lib/routes/linkresearcher/templates/bilingual.art

@@ -0,0 +1,7 @@
+{{ each en }}
+{{ if $index !== 0 }}
+<br>
+{{ /if }}
+<p>{{ $value }}</p>
+<p>{{ zh[$index] }}</p>
+{{ /each }}

package/lib/routes/linkresearcher/types.ts

@@ -0,0 +1,103 @@
+interface BaseItem {
+    id: string;
+    title: string;
+    tags: string[];
+    onlineTime: number;
+    cover: string;
+}
+
+export interface InformationItem extends BaseItem {
+    summary: string;
+}
+
+export interface ThesesItem extends BaseItem {
+    authors: string[];
+    content: string;
+    journals: string[];
+    publishDate: string;
+    source: {
+        sourceType: string;
+    };
+    subject: string;
+    thesisTitle: string;
+}
+
+export interface ArticleItem extends BaseItem {
+    columns: string[];
+    source: {
+        logo: string;
+        sourceId: string;
+        sourceName: string;
+        sourceType: string;
+    };
+    summary: string;
+    type: string;
+}
+
+export type SearchResultItem = InformationItem | ThesesItem | ArticleItem;
+
+export interface ThesesDetailResponse {
+    columns: string[];
+    content: string;
+    cover: string;
+    enTextList: string[];
+    id: string;
+    journals: string[];
+    link: string;
+    onlineTime: number;
+    original: boolean;
+    paperList: {
+        authors: string[];
+        checkname: string;
+        doi: string;
+        id: string;
+        journal: string;
+        link: string;
+        publishDate: string;
+        subjects: string[];
+        summary: string;
+        title: string;
+        translateSummary: string;
+        type: string;
+    }[];
+    relevant: {
+        timestamp: number;
+        type: string;
+    }[];
+    source: {
+        sourceId: string;
+        sourceName: string;
+    };
+    sourceKey: string;
+    sourceType: string;
+    tags: string[];
+    template: boolean;
+    title: string;
+    userType: number;
+    zhTextList?: string[];
+}
+
+export interface InformationDetailResponse {
+    columns: string[];
+    content: string;
+    cover: string;
+    id: string;
+    onlineTime: number;
+    original: boolean;
+    relevant: {
+        timestamp: number;
+        type: string;
+    }[];
+    source: {
+        sourceId: string;
+        sourceName: string;
+    };
+    sourceKey: string;
+    subject: string;
+    summary: string;
+    tags: string[];
+    title: string;
+    type: string;
+}
+
+export type DetailResponse = ThesesDetailResponse | InformationDetailResponse;

package/lib/routes/liulinblog/itnews.ts

@@ -9,5 +9,5 @@ export const route: Route = {
 function handler(ctx) {
     const { channel } = ctx.req.param();
     const redirectTo = `/liulinblog/${channel}`;
-    ctx.redirect(redirectTo);
+    ctx.set('redirect', redirectTo);
 }

package/lib/routes/lofter/collection.ts

@@ -6,7 +6,7 @@ import { parseDate } from '@/utils/parse-date';
 
 export const route: Route = {
     path: '/collection/:collectionID',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/lofter/collection/552041',
     parameters: { collectionID: 'Lofter collection ID, can be found in the share URL' },
     features: {

package/lib/routes/lofter/tag.ts

@@ -6,7 +6,7 @@ import { JSDOM } from 'jsdom';
 
 export const route: Route = {
     path: '/tag/:name?/:type?',
-    categories: ['social-media'],
+    categories: ['social-media', 'popular'],
     example: '/lofter/tag/cosplay/date',
     parameters: { name: 'tag name, such as `名侦探柯南`, `摄影` by default', type: 'ranking type, see below, new by default' },
     features: {

package/lib/routes/logrocket/index.ts

@@ -0,0 +1,71 @@
+import { parseDate } from '@/utils/parse-date';
+import ofetch from '@/utils/ofetch'; // 统一使用的请求库
+import { load } from 'cheerio'; // 类似 jQuery 的 API HTML 解析器
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+// import { getCurrentPath } from '@/utils/helpers';
+// import { art } from '@/utils/render';
+// import path from 'node:path';
+// const __dirname = getCurrentPath(import.meta.url);
+export const route: Route = {
+    path: '/:type',
+    categories: ['blog'],
+    example: '/logrocket/dev',
+    parameters: { type: 'dev | product-management | ux-design' },
+    radar: [
+        {
+            source: ['blog.logrocket.com'],
+        },
+    ],
+    name: 'blog.logrocket',
+    maintainers: ['findwei'],
+    handler,
+    url: 'blog.logrocket.com/',
+};
+async function handler(ctx) {
+    const type = ctx.req.param('type');
+    const link = 'https://blog.logrocket.com/';
+    let title = 'Dev';
+    if (type === 'product-management') {
+        title = 'Product Management';
+    } else if (type === 'ux-design') {
+        title = 'UX Design';
+    }
+    const response = await ofetch(`${link}${type}`);
+    const $ = load(response);
+    const list = $('div.post-list .post-card')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('a').first();
+            const title = item.find('.post-card-title').first();
+            return {
+                title: title.text(),
+                link: a.attr('href'),
+                pubDate: parseDate(item.find('.post-card-author-name').next().text().split(' ⋅ ')[0], 'MMM D, YYYY'),
+                author: item.find('.post-card-author-name').text(),
+            };
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await ofetch(item.link);
+                const $ = load(response);
+                //
+                $('div.content-max-width .sidebar-container div.code-block').remove();
+                item.description = $('div.content-max-width .sidebar-container').html();
+                // item.description = art(path.join(__dirname, 'templates/description.art'), {
+                //     // header: $('#post-header').html(),
+                //     description: $('div.content-max-width .the-content-container').remove('.lr-content div.code-block.code-block-77').remove('.lr-content .code-block.code-block-57').html(),
+                // });
+                return item;
+            })
+        )
+    );
+    return {
+        title: `logrocket-${title}`,
+        link,
+        description: `logrocket-${title}`,
+        item: items,
+    };
+}

package/lib/routes/logrocket/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'logrocket blog',
+    url: 'blog.logrocket.com',
+    lang: 'en',
+};

package/lib/routes/m-78/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '円谷ステーション',
+    url: 'm-78.jp',
+    lang: 'ja',
+};