rsshub 1.0.0-master.fd5a888 → 1.0.0-master.fd5cc68

package/lib/routes/douban/other/list.ts

@@ -30,31 +30,31 @@ export const route: Route = {
     maintainers: ['5upernova-heng', 'honue'],
     handler,
     description: `| 榜单 / 集合        | 路由                          |
-  | ------------------ | ----------------------------- |
-  | 实时热门书影音     | subject\_real\_time\_hotest   |
-  | 影院热映           | movie\_showing                |
-  | 实时热门电影       | movie\_real\_time\_hotest     |
-  | 实时热门电视       | tv\_real\_time\_hotest        |
-  | 一周口碑电影榜     | movie\_weekly\_best           |
-  | 华语口碑剧集榜     | tv\_chinese\_best\_weekly     |
-  | 全球口碑剧集榜     | tv\_global\_best\_weekly      |
-  | 国内口碑综艺榜     | show\_chinese\_best\_weekly   |
-  | 国外口碑综艺榜     | show\_global\_best\_weekly    |
-  | 热播新剧国产剧     | tv\_domestic                  |
-  | 热播新剧欧美剧     | tv\_american                  |
-  | 热播新剧日剧       | tv\_japanese                  |
-  | 热播新剧韩剧       | tv\_korean                    |
-  | 热播新剧动画       | tv\_animation                 |
-  | 虚构类小说热门榜   | book\_fiction\_hot\_weekly    |
-  | 非虚构类小说热门榜 | book\_nonfiction\_hot\_weekly |
-  | 热门单曲榜         | music\_single                 |
-  | 华语新碟榜         | music\_chinese                |
-  | ...                | ...                           |
+| ------------------ | ----------------------------- |
+| 实时热门书影音     | subject\_real\_time\_hotest   |
+| 影院热映           | movie\_showing                |
+| 实时热门电影       | movie\_real\_time\_hotest     |
+| 实时热门电视       | tv\_real\_time\_hotest        |
+| 一周口碑电影榜     | movie\_weekly\_best           |
+| 华语口碑剧集榜     | tv\_chinese\_best\_weekly     |
+| 全球口碑剧集榜     | tv\_global\_best\_weekly      |
+| 国内口碑综艺榜     | show\_chinese\_best\_weekly   |
+| 国外口碑综艺榜     | show\_global\_best\_weekly    |
+| 热播新剧国产剧     | tv\_domestic                  |
+| 热播新剧欧美剧     | tv\_american                  |
+| 热播新剧日剧       | tv\_japanese                  |
+| 热播新剧韩剧       | tv\_korean                    |
+| 热播新剧动画       | tv\_animation                 |
+| 虚构类小说热门榜   | book\_fiction\_hot\_weekly    |
+| 非虚构类小说热门榜 | book\_nonfiction\_hot\_weekly |
+| 热门单曲榜         | music\_single                 |
+| 华语新碟榜         | music\_chinese                |
+| ...                | ...                           |
 
-  | 额外参数 | 含义                   | 接受的值 | 默认值 |
-  | -------- | ---------------------- | -------- | ------ |
-  | playable | 仅看有可播放片源的影片 | 0/1      | 0      |
-  | score    | 筛选评分               | 0-10     | 0      |
+| 额外参数 | 含义                   | 接受的值 | 默认值 |
+| -------- | ---------------------- | -------- | ------ |
+| playable | 仅看有可播放片源的影片 | 0/1      | 0      |
+| score    | 筛选评分               | 0-10     | 0      |
 
   用例：\`/douban/list/tv_korean/playable=1&score=8\`
 

package/lib/routes/douban/other/recommended.ts

@@ -24,15 +24,15 @@ export const route: Route = {
     maintainers: ['honue'],
     handler,
     description: `| 额外参数 | 含义                   | 接受的值 | 默认值 |
-  | -------- | ---------------------- | -------- | ------ |
-  | playable | 仅看有可播放片源的影片 | 0/1      | 0      |
-  | score    | 筛选评分               | 0-10     | 0      |
+| -------- | ---------------------- | -------- | ------ |
+| playable | 仅看有可播放片源的影片 | 0/1      | 0      |
+| score    | 筛选评分               | 0-10     | 0      |
 
   用例：\`/douban/recommended/tv/playable=0&score=8\`
 
-  :::tip
+::: tip
   整合了 /douban/list/ 路由，省去每月手动更新 id 参数，因为当月推荐剧集片单中，会有还未播出 / 开评分剧集、海外平台播出剧集，请自行考虑是否使用额外参数。
-  :::`,
+:::`,
 };
 
 async function handler(ctx) {

package/lib/routes/douban/other/weekly-best.ts

@@ -23,8 +23,8 @@ export const route: Route = {
     maintainers: ['numm233', 'nczitzk'],
     handler,
     description: `| 一周口碑电影榜      | 华语口碑剧集榜            |
-  | ------------------- | ------------------------- |
-  | movie\_weekly\_best | tv\_chinese\_best\_weekly |`,
+| ------------------- | ------------------------- |
+| movie\_weekly\_best | tv\_chinese\_best\_weekly |`,
 };
 
 async function handler(ctx) {

package/lib/routes/douban/people/status.ts

@@ -22,21 +22,21 @@ export const route: Route = {
 
 对于豆瓣用户广播内容，在 \`routeParams\` 参数中以 query string 格式设置如下选项可以控制输出的样式
 
-  | 键                         | 含义                                                           | 接受的值       | 默认值 |
-  | -------------------------- | -------------------------------------------------------------- | -------------- | ------ |
-  | readable                   | 是否开启细节排版可读性优化                                     | 0/1/true/false | false  |
-  | authorNameBold             | 是否加粗作者名字                                               | 0/1/true/false | false  |
-  | showAuthorInTitle          | 是否在标题处显示作者                                           | 0/1/true/false | true   |
-  | showAuthorInDesc           | 是否在正文处显示作者                                           | 0/1/true/false | false  |
-  | showAuthorAvatarInDesc     | 是否在正文处显示作者头像（若阅读器会提取正文图片，不建议开启） | 0/1/true/false | false  |
-  | showEmojiForRetweet        | 显示 “🔁” 取代 “Fw”（转发）                                    | 0/1/true/false | false  |
-  | showRetweetTextInTitle     | 在标题出显示转发评论（置为 false 则在标题只显示被转发的广播）  | 0/1/true/false | false  |
-  | addLinkForPics             | 为图片添加可点击的链接                                         | 0/1/true/false | false  |
-  | showTimestampInDescription | 在正文处显示广播的时间戳                                       | 0/1/true/false | false  |
-  | showComments               | 在正文处显示评论                                               | 0/1/true/false | false  |
-  | widthOfPics                | 广播配图宽（生效取决于阅读器）                                 | 不指定 / 数字  | 不指定 |
-  | heightOfPics               | 广播配图高（生效取决于阅读器）                                 | 不指定 / 数字  | 不指定 |
-  | sizeOfAuthorAvatar         | 作者头像大小                                                   | 数字           | 48     |
+| 键                         | 含义                                                           | 接受的值       | 默认值 |
+| -------------------------- | -------------------------------------------------------------- | -------------- | ------ |
+| readable                   | 是否开启细节排版可读性优化                                     | 0/1/true/false | false  |
+| authorNameBold             | 是否加粗作者名字                                               | 0/1/true/false | false  |
+| showAuthorInTitle          | 是否在标题处显示作者                                           | 0/1/true/false | true   |
+| showAuthorInDesc           | 是否在正文处显示作者                                           | 0/1/true/false | false  |
+| showAuthorAvatarInDesc     | 是否在正文处显示作者头像（若阅读器会提取正文图片，不建议开启） | 0/1/true/false | false  |
+| showEmojiForRetweet        | 显示 “🔁” 取代 “Fw”（转发）                                    | 0/1/true/false | false  |
+| showRetweetTextInTitle     | 在标题出显示转发评论（置为 false 则在标题只显示被转发的广播）  | 0/1/true/false | false  |
+| addLinkForPics             | 为图片添加可点击的链接                                         | 0/1/true/false | false  |
+| showTimestampInDescription | 在正文处显示广播的时间戳                                       | 0/1/true/false | false  |
+| showComments               | 在正文处显示评论                                               | 0/1/true/false | false  |
+| widthOfPics                | 广播配图宽（生效取决于阅读器）                                 | 不指定 / 数字  | 不指定 |
+| heightOfPics               | 广播配图高（生效取决于阅读器）                                 | 不指定 / 数字  | 不指定 |
+| sizeOfAuthorAvatar         | 作者头像大小                                                   | 数字           | 48     |
 
   指定更多与默认值不同的参数选项可以改善 RSS 的可读性，如
 

package/lib/routes/douban/people/wish.ts

@@ -22,9 +22,9 @@ export const route: Route = {
     handler,
     description: `对于豆瓣用户想看的内容，在 \`routeParams\` 参数中以 query string 格式设置如下选项可以控制输出的样式
 
-  | 键         | 含义       | 接受的值 | 默认值 |
-  | ---------- | ---------- | -------- | ------ |
-  | pagesCount | 查询页面数 |          | 1      |`,
+| 键         | 含义       | 接受的值 | 默认值 |
+| ---------- | ---------- | -------- | ------ |
+| pagesCount | 查询页面数 |          | 1      |`,
 };
 
 async function handler(ctx) {

package/lib/routes/douyin/live.ts

@@ -58,7 +58,7 @@ async function handler(ctx) {
             await page.goto(pageUrl, {
                 waitUntil: 'networkidle2',
             });
-            browser.close();
+            await browser.close();
 
             return roomInfo;
         },

package/lib/routes/douyin/namespace.ts

@@ -3,7 +3,7 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: '抖音直播',
     url: 'douyin.com',
-    description: `:::warning
+    description: `::: warning
 反爬严格，需要启用 puppeteer。\
 抖音的视频 CDN 会验证 Referer，意味着许多阅读器都无法直接播放内嵌视频，以下是一些变通解决方案：
 

package/lib/routes/douyin/types.ts

@@ -14,6 +14,7 @@ interface ShareInfo {
     share_title_myself: string;
     share_title_other: string;
     share_desc_info: string;
+    share_link_desc: string;
 }
 
 interface Author {
@@ -422,11 +423,6 @@ interface TextExtra {
     search_query_id: string;
 }
 
-interface ShareInfo {
-    share_url: string;
-    share_link_desc: string;
-}
-
 interface RiskInfos {
     vote: boolean;
     warn: boolean;

package/lib/routes/douyin/user.ts

@@ -69,7 +69,7 @@ async function handler(ctx) {
                 waitUntil: 'networkidle2',
             });
 
-            browser.close();
+            await browser.close();
 
             if (!postData) {
                 throw new Error('Empty post data. The request may be filtered by WAF.');

package/lib/routes/douyu/group.ts

@@ -31,8 +31,8 @@ export const route: Route = {
     maintainers: ['nczitzk'],
     handler,
     description: `| 回复时间排序 | 发布时间排序 |
-  | ------------ | ------------ |
-  | 1            | 2            |`,
+| ------------ | ------------ |
+| 1            | 2            |`,
 };
 
 async function handler(ctx) {

package/lib/routes/dtcj/datahero.ts

@@ -28,8 +28,8 @@ export const route: Route = {
     maintainers: ['nczitzk'],
     handler,
     description: `| 侠创 | 纽约数据科学学院 | RS 实验所 | 阿里云天池 |
-  | ---- | ---------------- | --------- | ---------- |
-  | 5    | 6                | 9         | 10         |`,
+| ---- | ---------------- | --------- | ---------- |
+| 5    | 6                | 9         | 10         |`,
 };
 
 async function handler(ctx) {

package/lib/routes/dtcj/datainsight.ts

@@ -28,8 +28,8 @@ export const route: Route = {
     handler,
     url: 'dtcj.com/dtcj/datainsight',
     description: `| 城数 | NEXT 情报局 | 专业精选 |
-  | ---- | ----------- | -------- |
-  | 3    | 1           | 4        |`,
+| ---- | ----------- | -------- |
+| 3    | 1           | 4        |`,
 };
 
 async function handler(ctx) {

package/lib/routes/duckdb/news.ts

@@ -36,8 +36,9 @@ async function handler() {
             item = $(item);
             return {
                 title: item.find('h3').text().trim(),
-                link: `https://duckdb.org${item.find('a').eq(2).attr('href')}`,
+                link: `https://duckdb.org${item.find('a.blocklink').attr('href')}`,
                 pubDate: timezone(parseDate(item.find('.date').text(), 'YYYY-MM-DD'), 0),
+                author: item.find('.author').text().trim(),
             };
         });
 
@@ -46,9 +47,7 @@ async function handler() {
             cache.tryGet(item.link, async () => {
                 const response = await got(item.link);
                 const $ = load(response.body);
-                item.author = $('.author').text();
-                item.description = $('.singleentry').html();
-
+                item.description = $('.contentwidth').find('h1, .infoline').remove().end().html();
                 // 上面每个列表项的每个属性都在此重用，
                 // 并增加了一个新属性“description”
                 return item;

package/lib/routes/dw/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'DW Deutsche Welle',
+    url: 'dw.com',
+};

package/lib/routes/dw/news.ts

@@ -0,0 +1,89 @@
+import { Route } from '@/types';
+import { processItems } from './utils';
+import got from '@/utils/got';
+import cache from '@/utils/cache';
+import { config } from '@/config';
+
+export const route: Route = {
+    path: '/news/:lang?/:id?',
+    categories: ['traditional-media'],
+    example: '/dw/news',
+    parameters: {
+        lang: 'Language, see below, default to en',
+        id: 'Category ID, see below, default to the id of the Top Stories Page of the language chosen',
+    },
+    features: {
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+        requireConfig: false,
+    },
+    name: 'News',
+    maintainers: ['quiniapiezoelectricity'],
+    handler,
+    description: `
+::: tip
+Parameters can be obtained from the official website, for instance:
+For the site https://www.dw.com/de/deutschland/s-12321 the language code would be \`de\` and the category ID would be \`s-1432\`.
+:::
+`,
+    radar: [
+        {
+            source: ['www.dw.com/:lang/:name/:id'],
+            target: '/news/:lang/:id',
+        },
+    ],
+};
+
+const defaultUrl = `https://www.dw.com/graph-api/en/content/navigation/9097`;
+const typenames = new Set(['Article', 'Liveblog', 'Video']);
+
+async function handler(ctx) {
+    const lang = ctx.req.param('lang') ?? 'en';
+    let id = ctx.req.param('id');
+
+    if (/^s-\d+$/.test(id)) {
+        id = id.match(/^s-(\d+)$/i)[1]; // convert s-1234 id to 1234
+    } else if (id === undefined) {
+        // Look up the id of the Top Stories Page of the selected language if id is not specified in the URL.
+        const navigation = await cache.tryGet(
+            'dw:navigation',
+            async () => {
+                const res = await got(defaultUrl);
+                return res.data.data.content.topStoriesNavigations;
+            },
+            config.cache.routeExpire,
+            false
+        );
+        id = navigation
+            .map((item) => item.namedUrl.split('/'))
+            .find((item) => item[1] === lang)[3]
+            .match(/^s-(\d+)$/i)[1];
+    }
+
+    const response = await got(`https://www.dw.com/graph-api/${lang}/content/navigation/${id}`);
+    const feed = response.data.data.content;
+    cache.set('dw:navigation', feed.topStoriesNavigations, config.cache.routeExpire);
+
+    const list = feed.contentComposition.informationSpaces.flatMap((section) => Object.values(section).flatMap((component) => component[0]?.contents || [])).filter((item) => typenames.has(item.__typename) && item.id);
+    const items = await processItems(
+        list.map((item) => {
+            item.link = new URL(item.namedUrl, 'https://www.dw.com').href;
+            item.pubDate = item.contentDate;
+            item.description = item.teaser;
+            item.language = lang;
+            item.type = item.__typename.toLowerCase();
+            return item;
+        })
+    );
+
+    return {
+        title: `DW | ${feed.title}`,
+        link: feed.canonicalUrl,
+        description: feed.metaDescription,
+        language: feed.topStoriesNavigations.find((item) => item.namedUrl.startsWith(`/${lang}/`))?.localeLang ?? lang,
+        item: items,
+    };
+}

package/lib/routes/dw/rss.ts

@@ -0,0 +1,62 @@
+import { Route } from '@/types';
+import { config } from '@/config';
+import Parser from 'rss-parser';
+import { processItems } from './utils';
+
+export const route: Route = {
+    path: '/rss/:channel?',
+    categories: ['traditional-media'],
+    example: '/dw/rss/rss-en-all',
+    parameters: {
+        category: 'RSS Feed Channel, see below, `rss-en-all` by default',
+    },
+    features: {
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+        requireConfig: false,
+    },
+    name: 'RSS',
+    maintainers: ['quiniapiezoelectricity'],
+    handler,
+    description: `
+For a full list of RSS Feed Channels in English, please refer to [DW RSS Feeds](https://corporate.dw.com/en/rss-feeds/a-68693346).
+RSS Feed Channels in other languages are also available, for example: \`rss-chi-all\` renders the RSS feed in Chinese and \`rss-de-all\` for the RSS Feed in German 
+`,
+};
+
+async function handler(ctx) {
+    const category = ctx.req.param('channel') ?? 'rss-en-all';
+
+    const parser = new Parser({
+        customFields: {
+            item: ['dwsyn:contentID'],
+        },
+        headers: {
+            'User-Agent': config.ua,
+        },
+    });
+
+    const feed = await parser.parseURL(`https://rss.dw.com/rdf/${category}`);
+    const items = await processItems(
+        feed.items.map((item) => {
+            item.id = item['dwsyn:contentID'];
+            item.pubDate = item.isoDate;
+            item.description = item.content;
+            const link = new URL(item.link);
+            link.search = '';
+            item.link = link.href;
+            item.type = link.pathname.substring(link.pathname.lastIndexOf('/') + 1).startsWith('live-') ? 'liveblog' : 'article'; // dw rss feed only includes liveblogs and articles
+            return item;
+        })
+    );
+
+    return {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: items,
+    };
+}

package/lib/routes/dw/templates/description.art

@@ -0,0 +1,23 @@
+{{ if teaser }}
+    <blockquote><p><em>{{ teaser }}</em></p></blockquote>
+{{ /if }}
+{{ if video }}
+    {{@ video }}
+{{ else if mainImage }}
+    <figure>
+        <img 
+            src="https://static.dw.com/image/{{ mainImage.targetId }}_{{ formatId }}.jpg" 
+            alt="{{ mainImage.additionalInformation }}" 
+        >
+        <figcaption>
+            {{ mainImage.description }}
+            <small>{{ imageI18n }}: {{ mainImage.target.licenserSupplement }}</small>
+        </figcaption>
+    </figure>
+{{ /if }}
+{{ if text }}
+   {{@ text }}
+{{ /if }}
+{{ if liveblog }}
+    {{@ liveblog }}
+{{ /if }}

package/lib/routes/dw/templates/liveblog.art

@@ -0,0 +1,13 @@
+{{ if posts }}
+{{ each posts }}
+    <hr>
+    {{ if $value.localizedContentDate }}<p><i>{{ $value.localizedContentDate }}</i></p>{{ /if }}
+    {{ if $value.title }}<h2>{{ $value.title }}</h2>{{ /if }}
+    {{ if $value.persons }}
+        {{ each $value.persons }}
+        <p><i>{{ $value.fullName }}</i></p>
+        {{ /each }}
+    {{ /if }}
+    {{ if $value.text }}{{@ $value.text }}{{ /if }}
+{{ /each }}
+{{ /if }}

package/lib/routes/dw/templates/video.art

@@ -0,0 +1,14 @@
+{{ if hlsVideoSrc }}
+<video
+{{ if posterImageUrl }} poster="{{ posterImageUrl }}" {{ /if }}
+controls preload="metadata">
+    <source
+    src="{{ hlsVideoSrc }}"
+    type="application/x-mpegURL"
+    >
+    <source
+    src="{{ mp4VideoSrc }}"
+    type="video/mp4"
+    >
+</video>
+{{ /if }}

package/lib/routes/dw/utils.ts

@@ -0,0 +1,172 @@
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load, type CheerioAPI } from 'cheerio';
+import { art } from '@/utils/render';
+import path from 'node:path';
+import { getCurrentPath } from '@/utils/helpers';
+
+const __dirname = getCurrentPath(import.meta.url);
+const formatId = '605';
+
+const i18n = (word: string, lang: string) => {
+    switch (word) {
+        case 'Image':
+            switch (lang) {
+                case 'sq':
+                    return 'Fotografi';
+                case 'am':
+                    return 'ምስል';
+                case 'ar':
+                    return 'صورة من';
+                case 'bn':
+                    return 'ছবি';
+                case 'bs':
+                    return 'Foto';
+                case 'bg':
+                    return 'Снимка';
+                case 'zh':
+                    return '图像来源';
+                case 'zh-hant':
+                    return '圖片來源';
+                case 'hr':
+                    return 'Foto';
+                case 'fa-af':
+                    return 'عکس';
+                case 'en':
+                    return 'Image';
+                case 'fr':
+                    return 'Image';
+                case 'de':
+                    return 'Bild';
+                case 'el':
+                    return 'Εικόνα';
+                case 'ha':
+                    return 'Hoto';
+                case 'hi':
+                    return 'तस्वीर';
+                case 'id':
+                    return 'Foto';
+                case 'sw':
+                    return 'Picha';
+                case 'mk':
+                    return 'Фотографија';
+                case 'ps':
+                    return 'انځور';
+                case 'fa-ir':
+                    return 'عکس';
+                case 'pl':
+                    return 'Zdjęcie';
+                case 'pt-002':
+                    return 'Foto';
+                case 'pt-br':
+                    return 'Foto';
+                case 'ro':
+                    return 'Imagine';
+                case 'ru':
+                    return 'Фото';
+                case 'sr':
+                    return 'Foto';
+                case 'es':
+                    return 'Imagen';
+                case 'tr':
+                    return 'Fotoğraf';
+                case 'uk':
+                    return 'Фото';
+                case 'ur':
+                    return 'تصویر';
+                default:
+                    return 'Image';
+            }
+        default:
+            return word;
+    }
+};
+
+const m3u8tomp4 = (src: string) =>
+    src.replace('https://hlsvod.dw.com/i/', 'https://tvdownloaddw-a.akamaihd.net/').replace(',AVC_480x270,AVC_512x288,AVC_640x360,AVC_960x540,AVC_1280x720,AVC_1920x1080,.mp4.csmil/master.m3u8', 'AVC_1920x1080.mp4');
+
+const processHtml = ($: CheerioAPI, contentLinks) => {
+    $('img').each((_, elem) => {
+        try {
+            const id = $(elem).attr('data-id');
+            const contentLink = contentLinks.find((item) => String(item.targetId) === id);
+            $(elem).attr({
+                title: contentLink?.name,
+                alt: contentLink?.description,
+                src: `https://static.dw.com/image/${id}_${formatId}.jpg`,
+            });
+            $(elem).removeAttr('style');
+        } catch {
+            // no-empty
+        }
+    });
+    $('video').each((_, elem) => {
+        try {
+            $(elem).attr('poster', $(elem).attr('data-posterurl'));
+        } catch {
+            // no-empty
+        }
+    });
+    $('iframe').each((_, elem) => {
+        try {
+            $(elem).attr('src', $(elem).attr('data-src'));
+        } catch {
+            // no-empty
+        }
+    });
+    $('svg').remove(); // svg will screw up in a lot of rss readers
+};
+
+const processContent = (item, content) => {
+    const $text = load(content.text);
+    processHtml($text, content.contentLinks);
+    const liveblog =
+        item.type === 'liveblog' && content.posts
+            ? art(path.join(__dirname, 'templates/liveblog.art'), {
+                  posts: content.posts.map((post) => {
+                      const $post = load(post.text);
+                      processHtml($post, content.contentLinks);
+                      post.text = $post.html();
+                      return post;
+                  }),
+              })
+            : undefined;
+    const video =
+        item.type === 'video' && content.hlsVideoSrc
+            ? art(path.join(__dirname, 'templates/video.art'), {
+                  hlsVideoSrc: content.hlsVideoSrc,
+                  mp4VideoSrc: m3u8tomp4(content.hlsVideoSrc),
+                  posterImageUrl: content.posterImageUrl,
+              })
+            : undefined;
+    item.description = art(path.join(__dirname, 'templates/description.art'), {
+        teaser: content.teaser,
+        video,
+        mainImage: $text(`[data-id="${content.mainContentImageLink?.targetId}"]`).length === 0 ? content.mainContentImageLink : undefined,
+        // occasionally the text html already includes the main image, testing to see if an image with the same id exists
+        text: $text.html(),
+        liveblog,
+        imageI18n: i18n('Image', item.language),
+        formatId,
+    });
+    if (content.trackingCategories) {
+        item.category = content.trackingCategories;
+    }
+    if (content.firstPersonArray) {
+        item.author = content.firstPersonArray.map((person) => person.fullName).join(', ');
+    }
+    return item;
+};
+
+export const processItems = async (items) => {
+    items = await Promise.all(
+        items.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const response = await got(`https://www.dw.com/graph-api/${item.language}/content/${item.type}/${item.id}`);
+                const content = response.data.data.content;
+                return processContent(item, content);
+            })
+        )
+    );
+    return items;
+};