rsshub 1.0.0-master.fd741f1 → 1.0.0-master.fd8463a

package/lib/routes/tradingview/blog.ts

@@ -1,12 +1,10 @@
 import { Route } from '@/types';
-import { getCurrentPath } from '@/utils/helpers';
-const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
-import asyncPool from 'tiny-async-pool';
+import pMap from 'p-map';
 import { art } from '@/utils/render';
 import path from 'node:path';
 
@@ -28,7 +26,7 @@ async function handler(ctx) {
 
     const $ = load(response);
 
-    const items = $('article[id]')
+    const list = $('article[id]')
         .slice(0, limit)
         .toArray()
         .map((item) => {
@@ -57,48 +55,48 @@ async function handler(ctx) {
             };
         });
 
-    for await (const item of asyncPool(3, items, (item) =>
-        cache.tryGet(item.link, async () => {
-            const { data: detailResponse } = await got(item.link);
-
-            const content = load(detailResponse);
-
-            content('div.entry-content')
-                .find('img')
-                .each((_, e) => {
-                    content(e).replaceWith(
-                        art(path.join(__dirname, 'templates/description.art'), {
-                            image: {
-                                src: content(e)
-                                    .prop('src')
-                                    .replace(/-\d+x\d+\./, '.'),
-                                width: content(e).prop('width'),
-                                height: content(e).prop('height'),
-                            },
-                        })
-                    );
+    const items = await pMap(
+        list,
+        (item) =>
+            cache.tryGet(item.link, async () => {
+                const { data: detailResponse } = await got(item.link);
+
+                const content = load(detailResponse);
+
+                content('div.entry-content')
+                    .find('img')
+                    .each((_, e) => {
+                        content(e).replaceWith(
+                            art(path.join(__dirname, 'templates/description.art'), {
+                                image: {
+                                    src: content(e)
+                                        .prop('src')
+                                        .replace(/-\d+x\d+\./, '.'),
+                                    width: content(e).prop('width'),
+                                    height: content(e).prop('height'),
+                                },
+                            })
+                        );
+                    });
+
+                item.title = content('meta[property="og:title"]').prop('content');
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    image: {
+                        src: content('meta[property="og:image"]').prop('content'),
+                        alt: item.title,
+                    },
+                    description: content('div.entry-content').html(),
                 });
+                item.author = content('meta[property="og:site_name"]').prop('content');
+                item.category = content('div.sections a.section')
+                    .toArray()
+                    .map((c) => content(c).text());
+                item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY');
 
-            item.title = content('meta[property="og:title"]').prop('content');
-            item.description = art(path.join(__dirname, 'templates/description.art'), {
-                image: {
-                    src: content('meta[property="og:image"]').prop('content'),
-                    alt: item.title,
-                },
-                description: content('div.entry-content').html(),
-            });
-            item.author = content('meta[property="og:site_name"]').prop('content');
-            item.category = content('div.sections a.section')
-                .toArray()
-                .map((c) => content(c).text());
-            item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY');
-
-            return item;
-        })
-    )) {
-        items.shift();
-        items.push(item);
-    }
+                return item;
+            }),
+        { concurrency: 3 }
+    );
 
     const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href;
 

package/lib/routes/transcriptforest/index.ts

@@ -1,6 +1,4 @@
 import { Route } from '@/types';
-import { getCurrentPath } from '@/utils/helpers';
-const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
 import got from '@/utils/got';

package/lib/routes/transformer-circuits/index.ts

@@ -0,0 +1,119 @@
+import { Route, DataItem } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import path from 'node:path';
+import logger from '@/utils/logger';
+
+// Define the main route path
+export const route: Route = {
+    path: '/',
+    categories: ['programming'],
+    example: '/transformer-circuits',
+    parameters: {},
+    radar: [
+        {
+            source: ['transformer-circuits.pub/'],
+            target: '/',
+        },
+    ],
+    name: 'Articles',
+    maintainers: ['shinmohuang'],
+    handler,
+};
+
+async function handler() {
+    const rootUrl = 'https://transformer-circuits.pub';
+
+    // Fetch the main page
+    const response = await ofetch(rootUrl);
+    const $ = load(response);
+
+    // Get all article links and basic info
+    const articlePromises = $('.toc a')
+        .toArray()
+        .map((item) => {
+            const $item = $(item);
+            const currentElement = $item;
+            const dateElement = $item.prevAll('.date').first();
+            const currentDate = dateElement.text().trim();
+
+            // Check if this is an article (either paper or note)
+            if (currentElement.hasClass('paper') || currentElement.hasClass('note')) {
+                const articleType = currentElement.hasClass('paper') ? 'Paper' : 'Note';
+
+                // Extract title and metadata from the list page
+                const title = currentElement.find('h3').text().trim();
+                let author = '';
+                const byline = currentElement.find('.byline');
+                if (byline.length) {
+                    author = byline.text().trim();
+                }
+                const description = currentElement.find('.description').text().trim();
+                const href = currentElement.attr('href');
+                const articleUrl = href ? (href.startsWith('http') ? href : `${rootUrl}/${href}`) : rootUrl;
+
+                // Cache the whole article object instead of just the content
+                return cache.tryGet(articleUrl, async () => {
+                    const fullContent = await fetchArticleContent(articleUrl);
+                    return {
+                        title,
+                        link: articleUrl,
+                        pubDate: parseDate(currentDate, 'MMMM YYYY'),
+                        author,
+                        description: fullContent || `${articleType}: ${description}`,
+                        category: ['AI', 'Machine Learning', 'Anthropic', 'Transformer Circuits'],
+                    };
+                });
+            }
+            return null;
+        });
+
+    // Wait for all article fetches to complete
+    const articlesWithContent = (await Promise.all(articlePromises)).filter(Boolean) as DataItem[];
+
+    return {
+        title: 'Transformer Circuits Thread',
+        link: rootUrl,
+        item: articlesWithContent,
+        description: 'Research on reverse engineering transformer language models into human-understandable programs',
+    };
+}
+
+// Function to fetch and parse article content
+async function fetchArticleContent(url) {
+    try {
+        const response = await ofetch(url);
+        const $ = load(response);
+
+        // Remove navigation and other unnecessary elements
+        $('.article-header, .tooltip, modal, script, style, d-front-matter, .visual-toc').remove();
+
+        // Get the main content - d-article is the custom tag used by this site
+        let content = $('d-article').html();
+
+        // If d-article not found, try more specific fallback selectors
+        if (!content) {
+            // Try to find content in common article containers, but avoid selecting the entire body
+            content = $('main article, .article-content, .post-content, .content-area').html() || $('.content, .article, .post').html() || $('main').html();
+
+            // If still no content found, log a warning
+            if (!content) {
+                logger.warn(`No suitable content container found for ${url}`);
+                content = `<p>Could not extract content. Please visit <a href="${url}">the original page</a>.</p>`;
+            }
+        }
+
+        // Create an HTML fragment (not a full document) for the RSS description
+        return art(path.join(__dirname, 'templates/article.art'), {
+            content,
+            link: url,
+        });
+    } catch (error: unknown) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        logger.error(`Error fetching article content from ${url}: ${errorMessage}`);
+        return null; // Return null on error, we'll fall back to description
+    }
+}

package/lib/routes/transformer-circuits/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Transformer Circuits',
+    url: 'transformer-circuits.pub',
+    lang: 'en',
+};

package/lib/routes/transformer-circuits/templates/article.art

@@ -0,0 +1,62 @@
+<style>
+    .content-wrapper {
+        font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+        line-height: 1.6;
+        color: #333;
+    }
+    img {
+        max-width: 100%;
+        height: auto;
+    }
+    pre, code {
+        background-color: #f5f5f5;
+        border-radius: 3px;
+        padding: 0.2em 0.4em;
+        overflow-x: auto;
+    }
+    a {
+        color: #0366d6;
+        text-decoration: none;
+    }
+    a:hover {
+        text-decoration: underline;
+    }
+    h1, h2, h3, h4, h5, h6 {
+        margin-top: 24px;
+        margin-bottom: 16px;
+        font-weight: 600;
+        line-height: 1.25;
+    }
+    p, ul, ol {
+        margin-bottom: 16px;
+    }
+    .read-original {
+        margin-top: 30px;
+        margin-bottom: 30px;
+        text-align: center;
+        padding: 10px;
+        background-color: #f7f7f7;
+        border-radius: 4px;
+    }
+    /* Support for custom elements used on transformer-circuits website */
+    d-figure, figure {
+        margin: 20px 0;
+        text-align: center;
+    }
+    d-byline {
+        font-size: 0.9em;
+        color: #666;
+        margin: 15px 0;
+    }
+    .gdoc-image img {
+        max-width: 100%;
+        display: block;
+        margin: 0 auto;
+    }
+</style>
+<div class="content-wrapper">
+    {{@ content }}
+</div>
+<div class="read-original">
+    <a href="{{ link }}" target="_blank">Read Original</a>
+</div> 

package/lib/routes/tribalfootball/latest.ts

@@ -1,6 +1,4 @@
 import { Route } from '@/types';
-import { getCurrentPath } from '@/utils/helpers';
-const __dirname = getCurrentPath(import.meta.url);
 
 import cache from '@/utils/cache';
 import got from '@/utils/got';
@@ -29,7 +27,8 @@ async function handler() {
     const rss = await got(rssUrl);
     const $ = load(rss.data, { xmlMode: true });
     const items = $('rss > channel > item')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             const $item = $(item);
             let link = $item.find('link').text();
             link = new URL(link);
@@ -44,8 +43,7 @@ async function handler() {
                 author: $item.find(String.raw`dc\:creator`).text(),
                 _header_image: $item.find('enclosure').attr('url'),
             };
-        })
-        .get();
+        });
 
     await Promise.all(
         items.map((item) =>

package/lib/routes/trow/portal.ts

@@ -52,20 +52,16 @@ async function handler() {
         title: `The Ring of Wonder - Portal`,
         link: `https://trow.cc`,
         description: `The Ring of Wonder 首页更新`,
-        item:
-            list &&
-            list
-                .map((index, item) => {
-                    item = $(item);
-                    const dateraw = item.find('.postdetails').text();
-                    return {
-                        title: item.find('.maintitle p:nth-child(2) > a').text(),
-                        description: item.find('.portal_news_content .row18').html(),
-                        link: item.find('.maintitle p:nth-child(2) > a').attr('href'),
-                        author: item.find('.postdetails a').text(),
-                        pubDate: timezone(parseDate(dateraw.slice(3), 'YYYY-MM-DD, HH:mm'), +8),
-                    };
-                })
-                .get(),
+        item: list.toArray().map((item) => {
+            item = $(item);
+            const dateraw = item.find('.postdetails').text();
+            return {
+                title: item.find('.maintitle p:nth-child(2) > a').text(),
+                description: item.find('.portal_news_content .row18').html(),
+                link: item.find('.maintitle p:nth-child(2) > a').attr('href'),
+                author: item.find('.postdetails a').text(),
+                pubDate: timezone(parseDate(dateraw.slice(3), 'YYYY-MM-DD, HH:mm'), +8),
+            };
+        }),
     };
 }

package/lib/routes/tvb/news.ts

@@ -1,6 +1,4 @@
 import { Route } from '@/types';
-import { getCurrentPath } from '@/utils/helpers';
-const __dirname = getCurrentPath(import.meta.url);
 
 import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';

package/lib/routes/tver/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'TVer',
+    url: 'tver.jp',
+    lang: 'ja',
+};

package/lib/routes/tver/series.ts

@@ -0,0 +1,98 @@
+import { Route, Data, DataItem } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+import { Context } from 'hono';
+
+export const route: Route = {
+    path: '/series/:id',
+    categories: ['traditional-media'],
+    example: '/tver/series/srx2o7o3c8',
+    parameters: {
+        id: 'Series ID (as it appears in URLs). For example, in https://tver.jp/series/srx2o7o3c8, the ID is "srx2o7o3c8".',
+    },
+    radar: [
+        {
+            source: ['tver.jp/series/:id'],
+            target: '/series/:id',
+        },
+    ],
+    name: 'Series',
+    maintainers: ['yuikisaito'],
+    handler,
+};
+
+const commonHeaders = {
+    Accept: '*/*',
+    'Accept-Language': 'ja,en-US;q=0.7,en;q=0.3',
+    'Cache-Control': 'no-cache',
+    Pragma: 'no-cache',
+    'Sec-GPC': '1',
+    'Sec-Fetch-Dest': 'empty',
+    'Sec-Fetch-Mode': 'cors',
+    'Sec-Fetch-Site': 'same-site',
+};
+
+async function handler(ctx: Context): Promise<Data> {
+    const { id } = ctx.req.param();
+
+    const { result: browser } = await ofetch('https://platform-api.tver.jp/v2/api/platform_users/browser/create', {
+        method: 'POST',
+        body: 'device_type=pc',
+        headers: {
+            ...commonHeaders,
+            'Content-Type': 'application/x-www-form-urlencoded',
+        },
+        referrer: 'https://s.tver.jp/',
+        credentials: 'omit',
+        mode: 'cors',
+    });
+
+    const { platform_uid, platform_token } = browser;
+
+    const { title, description, broadcastProvider } = await ofetch(`https://statics.tver.jp/content/series/${id}.json`, {
+        method: 'GET',
+        headers: {
+            ...commonHeaders,
+        },
+        referrer: 'https://tver.jp/',
+        credentials: 'omit',
+        mode: 'cors',
+    });
+
+    const { result } = await ofetch(`https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/${id}?platform_uid=${platform_uid}&platform_token=${platform_token}`, {
+        method: 'GET',
+        headers: {
+            ...commonHeaders,
+            'x-tver-platform-type': 'web',
+        },
+        referrer: 'https://tver.jp/',
+        credentials: 'omit',
+        mode: 'cors',
+    });
+
+    const items: DataItem[] = (result.contents?.[0]?.contents ?? [])
+        .filter((i) => i.type === 'episode')
+        .map((i) => {
+            const rawPubDate = i.content.broadcastDateLabel;
+            const cleanedPubDate = rawPubDate.replaceAll(/\(.*?\)|放送分/g, '').trim();
+            const parsedPubDate = timezone(parseDate(cleanedPubDate, 'M月D日'), +9).toDateString();
+
+            return {
+                title: i.content.title,
+                link: `https://tver.jp/episodes/${i.content.id}`,
+                image: `https://statics.tver.jp/images/content/thumbnail/episode/xlarge/${i.content.id}.jpg`,
+                pubDate: parsedPubDate,
+            };
+        });
+
+    return {
+        title: 'TVer - ' + title,
+        description,
+        author: broadcastProvider.name,
+        link: `https://tver.jp/series/${id}`,
+        image: `https://statics.tver.jp/images/content/thumbnail/series/xlarge/${id}.jpg`,
+        language: 'ja',
+        item: items,
+    };
+}

package/lib/routes/tvtropes/featured.ts

@@ -1,6 +1,4 @@
 import { Route } from '@/types';
-import { getCurrentPath } from '@/utils/helpers';
-const __dirname = getCurrentPath(import.meta.url);
 
 import got from '@/utils/got';
 import { load } from 'cheerio';

package/lib/routes/twitch/live.ts

@@ -7,7 +7,7 @@ const TWITCH_CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko';
 
 export const route: Route = {
     path: '/live/:login',
-    categories: ['live', 'popular'],
+    categories: ['live'],
     view: ViewType.Notifications,
     example: '/twitch/live/riotgames',
     parameters: { login: 'Twitch username' },

package/lib/routes/twitch/video.ts

@@ -14,7 +14,7 @@ const FILTER_NODE_TYPE_MAP = {
 
 export const route: Route = {
     path: '/video/:login/:filter?',
-    categories: ['live', 'popular'],
+    categories: ['live'],
     view: ViewType.Videos,
     example: '/twitch/video/riotgames/highlights',
     parameters: {

package/lib/routes/twitter/api/mobile-api/login.ts

@@ -3,7 +3,7 @@
 import { bearerToken, guestActivateUrl } from './constants';
 import got from '@/utils/got';
 import ofetch from '@/utils/ofetch';
-import crypto from 'crypto';
+import crypto from 'node:crypto';
 import { v5 as uuidv5 } from 'uuid';
 import { authenticator } from 'otplib';
 import logger from '@/utils/logger';

package/lib/routes/twitter/api/web-api/login.ts

@@ -28,9 +28,7 @@ async function login({ username, password, authenticationSecret }) {
         await loginLimiterQueue.removeTokens(1);
 
         const cookieJar = new CookieJar();
-        const browser = await puppeteer({
-            stealth: true,
-        });
+        const browser = await puppeteer();
         const page = await browser.newPage();
         await page.goto('https://x.com/i/flow/login');
         await page.waitForSelector('input[autocomplete="username"]');

package/lib/routes/twitter/api/web-api/utils.ts

@@ -203,7 +203,7 @@ export const twitterGot = async (
                             }
                         }
                         logger.debug(`twitter debug: delete twitter cookie for token ${auth.token} with status ${response.status}, remaining tokens: ${config.twitter.authToken?.length}`);
-                        await cache.set(`${lockPrefix}${auth.token}`, '1', 86400);
+                        await cache.set(`${lockPrefix}${auth.token}`, '1', 3600);
                     }
                 } else {
                     logger.debug(`twitter debug: unlock twitter cookie with success for token ${auth.token}`);
@@ -248,7 +248,9 @@ export const paginationTweets = async (endpoint: string, userId: number | undefi
             return instructions.instructions;
         }
 
-        const instructions = data?.user?.result?.timeline_v2?.timeline?.instructions;
+        const userResult = data?.user?.result;
+        const timeline = userResult?.timeline?.timeline || userResult?.timeline?.timeline_v2 || userResult?.timeline_v2?.timeline;
+        const instructions = timeline?.instructions;
         if (!instructions) {
             logger.debug(`twitter debug: instructions not found in data: ${JSON.stringify(data)}`);
         }
@@ -297,11 +299,31 @@ export function gatherLegacyFromData(entries: any[], filterNested?: string[], us
                         continue;
                     }
                     t.legacy.user = t.core?.user_result?.result?.legacy || t.core?.user_results?.result?.legacy;
+                    // Add name and screen_name from core to maintain compatibility
+                    if (t.legacy.user && t.core?.user_results?.result?.core) {
+                        const coreUser = t.core.user_results.result.core;
+                        if (coreUser.name) {
+                            t.legacy.user.name = coreUser.name;
+                        }
+                        if (coreUser.screen_name) {
+                            t.legacy.user.screen_name = coreUser.screen_name;
+                        }
+                    }
                     t.legacy.id_str = t.rest_id; // avoid falling back to conversation_id_str elsewhere
                     const quote = t.quoted_status_result?.result?.tweet || t.quoted_status_result?.result;
                     if (quote) {
                         t.legacy.quoted_status = quote.legacy;
                         t.legacy.quoted_status.user = quote.core.user_result?.result?.legacy || quote.core.user_results?.result?.legacy;
+                        // Add name and screen_name from core for quoted status user
+                        if (t.legacy.quoted_status.user && quote.core?.user_results?.result?.core) {
+                            const quoteCoreUser = quote.core.user_results.result.core;
+                            if (quoteCoreUser.name) {
+                                t.legacy.quoted_status.user.name = quoteCoreUser.name;
+                            }
+                            if (quoteCoreUser.screen_name) {
+                                t.legacy.quoted_status.user.screen_name = quoteCoreUser.screen_name;
+                            }
+                        }
                     }
                     if (t.note_tweet) {
                         const tmp = t.note_tweet.note_tweet_results.result;

package/lib/routes/twitter/keyword.ts

@@ -4,7 +4,7 @@ import utils from './utils';
 
 export const route: Route = {
     path: '/keyword/:keyword/:routeParams?',
-    categories: ['social-media', 'popular'],
+    categories: ['social-media'],
     view: ViewType.SocialMedia,
     example: '/twitter/keyword/RSSHub',
     parameters: { keyword: 'keyword', routeParams: 'extra parameters, see the table above' },

package/lib/routes/twitter/list.ts

@@ -4,7 +4,7 @@ import utils from './utils';
 
 export const route: Route = {
     path: '/list/:id/:routeParams?',
-    categories: ['social-media', 'popular'],
+    categories: ['social-media'],
     example: '/twitter/list/1502570462752219136',
     parameters: { id: 'list id, get from url', routeParams: 'extra parameters, see the table above' },
     features: {

package/lib/routes/twitter/media.ts

@@ -5,7 +5,7 @@ import logger from '@/utils/logger';
 
 export const route: Route = {
     path: '/media/:id/:routeParams?',
-    categories: ['social-media', 'popular'],
+    categories: ['social-media'],
     view: ViewType.Pictures,
     example: '/twitter/media/_RSSHub',
     parameters: { id: 'username; in particular, if starts with `+`, it will be recognized as a [unique ID](https://github.com/DIYgod/RSSHub/issues/12221), e.g. `+44196397`', routeParams: 'extra parameters, see the table above.' },

package/lib/routes/twitter/namespace.ts

@@ -29,6 +29,7 @@ export const namespace: Namespace = {
 | \`forceWebApi\`                  | Force using Web API even if Developer API is configured, only available in \`/twitter/user\` and \`/twitter/keyword\`                    | \`0\`/\`1\`/\`true\`/\`false\` | \`false\`                                   |
 | \`count\`                        | \`count\` parameter passed to Twitter API, only available in \`/twitter/user\`                                                           | Unspecified/Integer    | Unspecified                               |
 | \`onlyMedia\`                    | Only get tweets with a media                                                                                                             | \`0\`/\`1\`/\`true\`/\`false\` | \`false\`                 |
+| \`mediaNumber \`                 | Number the medias                                                                                                            | \`0\`/\`1\`/\`true\`/\`false\` | \`false\`                 |
 
 Specify different option values than default values to improve readability. The URL
 

package/lib/routes/twitter/user.ts

@@ -5,7 +5,7 @@ import logger from '@/utils/logger';
 
 export const route: Route = {
     path: '/user/:id/:routeParams?',
-    categories: ['social-media', 'popular'],
+    categories: ['social-media'],
     view: ViewType.SocialMedia,
     example: '/twitter/user/_RSSHub',
     parameters: {