rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/tfc-taiwan/index.ts

@@ -1,47 +1,17 @@
 import { Route } from '@/types';
-import cache from '@/utils/cache';
-import got from '@/utils/got';
-import { load } from 'cheerio';
-import { baseUrl, parseList, parseItems } from './utils';
+import { handler } from './common';
 
 export const route: Route = {
-    path: ['/', '/category/:id{.+}', '/info', '/report', '/topic/:id'],
-    name: 'Unknown',
+    name: '最新相關資訊 / 最新查核報告',
     maintainers: ['TonyRL'],
+    example: '/tfc-taiwan',
+    path: '/:type?',
+    parameters: {
+        type: '分類，見下表，預設為 `report`',
+    },
     handler,
     url: 'tfc-taiwan.org.tw/articles/report',
-    url: 'tfc-taiwan.org.tw/articles/info',
+    description: `| 最新相關資訊 | 最新查核報告 |
+| ------------ | ------------ |
+| info         | report       |`,
 };
-
-async function handler(ctx) {
-    const requestPath = ctx.req.path;
-    const isTopic = requestPath.startsWith('/topic/');
-    let link = baseUrl;
-
-    if (isTopic) {
-        link += `/topic/${ctx.req.param('id')}`;
-    } else if (requestPath === '/') {
-        link += `/articles/report`;
-    } else {
-        link += `/articles${requestPath}`;
-    }
-
-    const { data: response } = await got(link);
-    const $ = load(response);
-
-    const list = $(`${isTopic ? '.view-grouping' : '.pane-clone-of-article'} .views-row-inner`)
-        .toArray()
-        .map((item) => parseList($(item)));
-
-    const items = await parseItems(list, cache.tryGet);
-
-    return {
-        title: $('head title').text(),
-        description: $('head meta[name="description"]').attr('content'),
-        image: $('head meta[property="og:image"]').attr('content'),
-        logo: $('head link[rel="shortcut icon"]').attr('href'),
-        icon: $('head link[rel="shortcut icon"]').attr('href'),
-        link,
-        item: items,
-    };
-}

package/lib/routes/tfc-taiwan/topic.ts

@@ -0,0 +1,17 @@
+import { Route } from '@/types';
+import { handler } from './common';
+
+export const route: Route = {
+    name: '專題 / 重點專區',
+    maintainers: ['TonyRL'],
+    example: '/tfc-taiwan/category/242',
+    path: '/:type/:id{.+}',
+    parameters: {
+        type: '分類，見下表，預設為 `report`',
+    },
+    handler,
+    url: 'tfc-taiwan.org.tw/articles/report',
+    description: `| 專題     | 重點專區 |
+| -------- | -------- |
+| category | topic    |`,
+};

package/lib/routes/the/index.ts

@@ -0,0 +1,214 @@
+import { Route } from '@/types';
+import { getCurrentPath } from '@/utils/helpers';
+const __dirname = getCurrentPath(import.meta.url);
+
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import path from 'node:path';
+
+import { apiSlug, bakeFilterSearchParams, bakeFiltersWithPair, bakeUrl, fetchData, getFilterParamsForUrl, parseFilterStr } from './util';
+import timezone from '@/utils/timezone';
+
+export const handler = async (ctx) => {
+    const { filter } = ctx.req.param();
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50;
+
+    const rootUrl = 'https://the.bi/s';
+    const filters = parseFilterStr(filter);
+    const filtersWithPair = await bakeFiltersWithPair(filters, rootUrl);
+
+    const searchParams = bakeFilterSearchParams(filters, 'name', false);
+    const apiSearchParams = bakeFilterSearchParams(filtersWithPair, 'id', true);
+
+    apiSearchParams.append('_embed', 'true');
+    apiSearchParams.append('per_page', String(limit));
+    apiSearchParams.append('page', '1');
+
+    const apiUrl = bakeUrl(`${apiSlug}/posts`, rootUrl, apiSearchParams);
+    const currentUrl = bakeUrl(getFilterParamsForUrl(filtersWithPair) ?? '', rootUrl, searchParams);
+
+    const { data: response } = await got(apiUrl);
+
+    const items = response.slice(0, limit).map((item) => {
+        const terminologies = item._embedded['wp:term'];
+        const guid = item.guid?.rendered ?? item.guid;
+
+        const $$ = load(item.content?.rendered ?? item.content);
+
+        const publication = $$("a[id='publication']").text(); // Must be obtained before being removed
+
+        const image = $$('img#poster').prop('data-srcset');
+
+        $$('figure.graf').each((_, el) => {
+            el = $$(el);
+
+            const imgEl = el.find('img');
+
+            el.replaceWith(
+                art(path.join(__dirname, 'templates/description.art'), {
+                    images: imgEl
+                        ? [
+                              {
+                                  src: imgEl.prop('src'),
+                                  width: imgEl.prop('width'),
+                                  height: imgEl.prop('height'),
+                              },
+                          ]
+                        : undefined,
+                })
+            );
+        });
+
+        const title = $$('h1').text();
+        const intro = $$('h2').text();
+
+        $$('h1').parent().remove();
+
+        const description = art(path.join(__dirname, 'templates/description.art'), {
+            images: image
+                ? [
+                      {
+                          src: image,
+                      },
+                  ]
+                : undefined,
+            intro,
+            description: $$.html(),
+        });
+
+        return {
+            title: item.title?.rendered ?? item.title ?? title,
+            description,
+            pubDate: timezone(parseDate(item.date_gmt), 0),
+            updated: timezone(parseDate(item.modified_gmt), 0),
+            link: item.link,
+            category: [...new Set(terminologies.flat().map((c) => c.name))],
+            author: [...item._embedded.author, { name: publication }],
+            guid,
+            id: guid,
+            content: {
+                html: description,
+                text: $$.text(),
+            },
+        };
+    });
+
+    const data = await fetchData(currentUrl, rootUrl);
+
+    return {
+        ...data,
+        item: items,
+    };
+};
+
+export const route: Route = {
+    path: '/:filter{.+}?',
+    name: '分类',
+    url: 'the.bi',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/the',
+    parameters: { filter: '过滤器，见下方描述' },
+    description: `:::tip
+  如果你想订阅特定类别或标签，可以在路由中填写 filter 参数。\`/category/rawmw7dsta2jew\` 可以实现订阅 [剩余价值](https://the.bi/s/rawmw7dsta2jew) 类别。此时，路由是 [\`/the/category/rawmw7dsta2jew/\`](https://rsshub.app/the/category/rawmw7dsta2jew).
+
+  你还可以订阅多个类别。\`/category/rawmw7dsta2jew,rawbcvxkktdkq8/\` 可以实现同时订阅 [剩余价值](https://the.bi/s/rawmw7dsta2jew) 和 [打江山](https://the.bi/s/rawbcvxkktdkq8) 两个类别。此时，路由是 [\`/the/category/rawmw7dsta2jew,rawbcvxkktdkq8\`](https://rsshub.app/the/category/rawmw7dsta2jew,rawbcvxkktdkq8).
+
+  类别和标签也可以合并订阅。\`/category/rawmw7dsta2jew/tag/raweekl3na8trq\` 订阅 [剩余价值](https://the.bi/s/rawmw7dsta2jew) 类别和 [动物](https://the.bi/s/raweekl3na8trq) 标签。此时，路由是 [\`/the/category/rawmw7dsta2jew/tag/raweekl3na8trq\`](https://rsshub.app/the/category/rawmw7dsta2jew/tag/raweekl3na8trq).
+
+  你还可以搜索关键字。\`/search/中国\` 搜索关键字 [中国](https://the.bi/s/?s=中国)。在这种情况下，路径是 [\`/the/search/中国\`](https://rsshub.app/the/search/中国).
+  :::
+
+  | 分类                                           | ID                                                               |
+  | ---------------------------------------------- | ---------------------------------------------------------------- |
+  | [时局图](https://the.bi/s/rawj7o4ypewv94)      | [rawj7o4ypewv94](https://rsshub.app/the/category/rawj7o4ypewv94) |
+  | [剩余价值](https://the.bi/s/rawmw7dsta2jew)    | [rawmw7dsta2jew](https://rsshub.app/the/category/rawmw7dsta2jew) |
+  | [打江山](https://the.bi/s/rawbcvxkktdkq8)      | [rawbcvxkktdkq8](https://rsshub.app/the/category/rawbcvxkktdkq8) |
+  | [中国经济](https://the.bi/s/raw4krvx85dh27)    | [raw4krvx85dh27](https://rsshub.app/the/category/raw4krvx85dh27) |
+  | [水深火热](https://the.bi/s/rawtn8jpsc6uvv)    | [rawtn8jpsc6uvv](https://rsshub.app/the/category/rawtn8jpsc6uvv) |
+  | [东升西降](https://the.bi/s/rawai5kd4z15il)    | [rawai5kd4z15il](https://rsshub.app/the/category/rawai5kd4z15il) |
+  | [大局 & 大棋](https://the.bi/s/raw2efkzejrsx8) | [raw2efkzejrsx8](https://rsshub.app/the/category/raw2efkzejrsx8) |
+  | [境外势力](https://the.bi/s/rawmpalhnlphuc)    | [rawmpalhnlphuc](https://rsshub.app/the/category/rawmpalhnlphuc) |
+  | [副刊](https://the.bi/s/rawxght2jr2u5z)        | [rawxght2jr2u5z](https://rsshub.app/the/category/rawxght2jr2u5z) |
+  | [天高地厚](https://the.bi/s/rawrsnh9zakqdx)    | [rawrsnh9zakqdx](https://rsshub.app/the/category/rawrsnh9zakqdx) |
+  | [Oyster](https://the.bi/s/rawdhl9hugdfn9)      | [rawdhl9hugdfn9](https://rsshub.app/the/category/rawdhl9hugdfn9) |
+  `,
+    categories: ['new-media'],
+
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['the.bi/s/:category?'],
+            target: (params) => {
+                const category = params.category;
+
+                return `/the${category ? `/category/${category}` : ''}`;
+            },
+        },
+        {
+            title: '时局图',
+            source: ['the.bi/s/rawj7o4ypewv94'],
+            target: '/category/rawj7o4ypewv94',
+        },
+        {
+            title: '剩余价值',
+            source: ['the.bi/s/rawmw7dsta2jew'],
+            target: '/category/rawmw7dsta2jew',
+        },
+        {
+            title: '打江山',
+            source: ['the.bi/s/rawbcvxkktdkq8'],
+            target: '/category/rawbcvxkktdkq8',
+        },
+        {
+            title: '中国经济',
+            source: ['the.bi/s/raw4krvx85dh27'],
+            target: '/category/raw4krvx85dh27',
+        },
+        {
+            title: '水深火热',
+            source: ['the.bi/s/rawtn8jpsc6uvv'],
+            target: '/category/rawtn8jpsc6uvv',
+        },
+        {
+            title: '东升西降',
+            source: ['the.bi/s/rawai5kd4z15il'],
+            target: '/category/rawai5kd4z15il',
+        },
+        {
+            title: '大局 & 大棋',
+            source: ['the.bi/s/raw2efkzejrsx8'],
+            target: '/category/raw2efkzejrsx8',
+        },
+        {
+            title: '境外势力',
+            source: ['the.bi/s/rawmpalhnlphuc'],
+            target: '/category/rawmpalhnlphuc',
+        },
+        {
+            title: '副刊',
+            source: ['the.bi/s/rawxght2jr2u5z'],
+            target: '/category/rawxght2jr2u5z',
+        },
+        {
+            title: '天高地厚',
+            source: ['the.bi/s/rawrsnh9zakqdx'],
+            target: '/category/rawrsnh9zakqdx',
+        },
+        {
+            title: 'Oyster',
+            source: ['the.bi/s/rawdhl9hugdfn9'],
+            target: '/category/rawdhl9hugdfn9',
+        },
+    ],
+};

package/lib/routes/the/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'The.bi',
+    url: 'the.bi',
+    categories: ['new-media'],
+    description: '',
+};

package/lib/routes/the/templates/description.art

@@ -0,0 +1,27 @@
+{{ if images }}
+  {{ each images image }}
+    {{ if image?.src }}
+      <figure>
+        <img
+          {{ if image.alt }}
+            alt="{{ image.alt }}"
+          {{ /if }}
+          {{ if image.width }}
+            alt="{{ image.width }}"
+          {{ /if }}
+          {{ if image.height }}
+            alt="{{ image.height }}"
+          {{ /if }}
+        src="{{ image.src }}">
+      </figure>
+    {{ /if }}
+  {{ /each }}
+{{ /if }}
+
+{{ if intro }}
+  <blockquote>{{ intro }}</blockquote>
+{{ /if }}
+
+{{ if description }}
+  {{@ description }}
+{{ /if }}

package/lib/routes/the/util.ts

@@ -0,0 +1,307 @@
+import got from '@/utils/got';
+import { load } from 'cheerio';
+
+const apiSlug = 'wp-json/wp/v2';
+
+interface Filter {
+    id: string;
+    name: string;
+    slug: string;
+}
+
+const filterKeys: Record<string, string> = {
+    search: 's',
+};
+
+const filterApiKeys: Record<string, string | undefined> = {
+    category: 'categories',
+    tag: 'tags',
+    search: undefined,
+};
+
+const filterApiKeysWithNoId = new Set(['search']);
+
+/**
+ * Bake filter search parameters.
+ *
+ * @param filterPairs - The filter pairs object.
+ *                      e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+ * @param pairKey - The filter pair key.
+ *                  e.g. `{ id: ..., name: ..., slug: ... }`.
+ * @param isApi - Indicates if the search parameters are for API.
+ * @returns The baked filter search parameters.
+ */
+const bakeFilterSearchParams = (filterPairs: Record<string, Filter[] | string[]>, pairKey: string, isApi: boolean = false): URLSearchParams => {
+    /**
+     * Bake filters recursively.
+     *
+     * @param filterPairs - The filter pairs object.
+     *                      e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+     * @param filterSearchParams - The filter search parameters.
+     *                             e.g. `category=a,b&tag=c`.
+     * @returns The baked filter search parameters.
+     *          e.g. `category=a,b&tag=c`.
+     */
+    const bakeFilters = (filterPairs: Record<string, Filter[] | string[]>, filterSearchParams: URLSearchParams): URLSearchParams => {
+        const keys = Object.keys(filterPairs).filter((key) => filterPairs[key]?.length > 0 && (isApi ? Object.hasOwn(filterApiKeys, key) : Object.hasOwn(filterKeys, key)));
+
+        if (keys.length === 0) {
+            return filterSearchParams;
+        }
+
+        const key = keys[0];
+        const pairs = filterPairs[key];
+
+        const originalFilters = { ...filterPairs };
+        delete originalFilters[key];
+
+        const filterKey = getFilterKeyForSearchParams(key, isApi);
+        const pairValues = pairs.map((pair) => (Object.hasOwn(pair, pairKey) ? pair[pairKey] : pair));
+
+        if (filterKey) {
+            filterSearchParams.append(filterKey, pairValues.join(','));
+        }
+
+        return bakeFilters(originalFilters, filterSearchParams);
+    };
+
+    return bakeFilters(filterPairs, new URLSearchParams());
+};
+
+/**
+ * Bake filters with pair.
+ *
+ * @param filters - The filters object.
+ *                  e.g. `{ category: [ a, b ], tag: [ c ] }`.
+ * @returns The baked filters.
+ *          e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+ */
+const bakeFiltersWithPair = async (filters: Record<string, string[]>, rootUrl: string) => {
+    /**
+     * Bake keywords recursively.
+     *
+     * @param key - The key.
+     *              e.g. `category` or `tag`.
+     * @param keywords - The keywords.
+     *                   e.g. `[ a, b ]`.
+     * @returns The baked keywords.
+     *          e.g. `[ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ]`.
+     */
+    const bakeKeywords = async (key: string, keywords: string[]) => {
+        if (keywords.length === 0) {
+            return [];
+        }
+
+        const [keyword, ...rest] = keywords;
+
+        const filter = await getFilterByKeyAndKeyword(key, keyword, rootUrl);
+
+        return [
+            ...(filter?.id && filter?.slug
+                ? [
+                      {
+                          id: filter.id,
+                          name: filter.name,
+                          slug: filter.slug,
+                      },
+                  ]
+                : []),
+            ...(await bakeKeywords(key, rest)),
+        ];
+    };
+
+    /**
+     * Bake filters recursively.
+     *
+     * @param filters - The filters object.
+     *                  e.g. `{ category: [ a, b ], tag: [ c ] }`.
+     * @param filtersWithPair - The filters with pairs.
+     *                          e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+     * @returns The baked filters.
+     *          e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+     */
+    const bakeFilters = async (filters: Record<string, string[]>, filtersWithPair: Record<string, Filter[]>) => {
+        const keys = Object.keys(filters);
+
+        if (keys.length === 0) {
+            return filtersWithPair;
+        }
+
+        const key = keys[0];
+        const keywords = filters[key];
+
+        const originalFilters = { ...filters };
+        delete originalFilters[key];
+
+        return bakeFilters(originalFilters, {
+            ...filtersWithPair,
+            [key]: filterApiKeysWithNoId.has(key) ? keywords : await bakeKeywords(key, keywords),
+        });
+    };
+
+    return await bakeFilters(filters, {});
+};
+
+/**
+ * Bake URL with search parameters.
+ *
+ * @param url - The URL.
+ * @param rootUrl - The root URL.
+ * @param searchParams - The search parameters.
+ * @returns The baked URL.
+ */
+const bakeUrl = (url: string, rootUrl: string, searchParams: URLSearchParams = new URLSearchParams()): string => {
+    const searchParamsStr = searchParams.toString();
+    const searchParamsSuffix = searchParamsStr ? `?${searchParamsStr}` : '';
+
+    return `${rootUrl}/${url}${searchParamsSuffix}`;
+};
+
+/**
+ * Fetch data from the specified URL.
+ *
+ * @param url - The URL to fetch data from.
+ * @param rootUrl - The root URL.
+ * @returns A promise that resolves to an object containing the fetched data to be added into `ctx.state.data`.
+ */
+const fetchData = async (url: string, rootUrl: string): Promise<object> => {
+    /**
+     * Request URLs recursively.
+     *
+     * @param urls - The URLs to request.
+     * @returns A promise that resolves to the response data or undefined if no response is available.
+     */
+    const requestUrls = async (urls: string[]): Promise<string | undefined> => {
+        if (urls.length === 0) {
+            return;
+        }
+
+        const [currentUrl, ...remainingUrls] = urls;
+        try {
+            const { data: response } = await got.get(currentUrl);
+            return response;
+        } catch {
+            return requestUrls(remainingUrls);
+        }
+    };
+
+    const response = await requestUrls([url, rootUrl]);
+
+    if (!response) {
+        return {};
+    }
+
+    const $ = load(response);
+
+    const title = $('title').first().text();
+    const image = new URL('wp-content/uploads/site_logo.png', rootUrl).href;
+
+    return {
+        title,
+        description: $('meta[property="og:description"]').attr('content') || $('meta[name="description"]').attr('content'),
+        link: url,
+        allowEmpty: true,
+        image,
+        author: $('meta[property="og:site_name"]').attr('content'),
+        language: $('html').attr('lang'),
+    };
+};
+
+/**
+ * Get filter by key and keyword.
+ *
+ * @param key - The key.
+ *              e.g. `category` or `tag`.
+ * @param keyword - The keywords.
+ *                  e.g. `keyword1`.
+ * @returns A promise that resolves to the filter object if found, or undefined if not found.
+ */
+const getFilterByKeyAndKeyword = async (key: string, keyword: string, rootUrl: string): Promise<Filter | undefined> => {
+    const apiFilterUrl = `${rootUrl}/${apiSlug}/${getFilterKeyForSearchParams(key, true)}`;
+
+    const { data: response } = await got(apiFilterUrl, {
+        searchParams: {
+            search: keyword,
+        },
+    });
+
+    return response.length > 0 ? response[0] : undefined;
+};
+
+/**
+ * Get filter key for search parameters.
+ *
+ * @param key - The key. e.g. `category` or `tag`.
+ * @param isApi - Indicates whether the key is for the API.
+ * @returns The filter key for search parameters, or undefined if not found.
+ *          e.g. `categories` or `tags`.
+ */
+const getFilterKeyForSearchParams = (key: string, isApi: boolean = false): string | undefined => {
+    const keys = isApi ? filterApiKeys : filterKeys;
+
+    return Object.hasOwn(keys, key) ? (keys[key] ?? key) : undefined;
+};
+
+/**
+ * Get filter parameters for URL.
+ *
+ * @param filterPairs - The filter pairs object.
+ *                      e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+ * @returns The filter parameters for the URL, or undefined if no filters are available.
+ */
+const getFilterParamsForUrl = (filterPairs: Record<string, Filter[]>): string | undefined => {
+    const keys = Object.keys(filterPairs).filter((key) => filterPairs[key].length > 0 && !Object.hasOwn(filterKeys, key));
+
+    if (keys.length === 0) {
+        return;
+    }
+
+    const key = keys[0];
+
+    return filterPairs[key].map((pair) => pair.slug).join('/');
+};
+
+/**
+ * Parses a filter string into a filters object.
+ *
+ * @param filterStr - The filter string to parse.
+ *                    e.g. `category/a,b/tag/c`.
+ * @returns The parsed filters object.
+ *          e.g. `{ category: [ 'a', 'b' ], tag: [ 'c' ] }`.
+ */
+const parseFilterStr = (filterStr: string | undefined): Record<string, string[]> => {
+    /**
+     * Recursively parses a filter string.
+     *
+     * @param remainingStr - The remaining filter string to parse.
+     *                       e.g. `category/a,b/tag/c`.
+     * @param filters - The accumulated filters object.
+     *                  e.g. `{ category: [ a, b ], tag: [ c ] }`.
+     * @param currentKey - The current filter key.
+     *                     e.g. `category` or `tag`.
+     * @returns The parsed filters object.
+     */
+    const parseStr = (remainingStr: string | undefined, filters: Record<string, string[]> = {}, currentKey?: string): Record<string, string[]> => {
+        if (!remainingStr) {
+            return filters;
+        }
+
+        const [word, ...rest] = remainingStr.split(/\/|,/);
+
+        const isKey = Object.hasOwn(filterApiKeys, word);
+        const key = isKey ? word : currentKey;
+
+        const newFilters = key
+            ? {
+                  ...filters,
+                  [key]: [...(filters[key] || []), ...(isKey ? [] : [word])],
+              }
+            : filters;
+
+        return parseStr(rest.join('/'), newFilters, key);
+    };
+
+    return parseStr(filterStr, {});
+};
+
+export { apiSlug, bakeFilterSearchParams, bakeFiltersWithPair, bakeUrl, fetchData, getFilterParamsForUrl, parseFilterStr };