rsshub 1.0.0-master.ff3a9ce → 1.0.0-master.ff3ea5a

package/lib/routes/missav/new.ts

@@ -20,6 +20,7 @@ export const route: Route = {
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
+        nsfw: true,
     },
     radar: [
         {
@@ -39,7 +40,7 @@ async function handler() {
     const baseUrl = 'https://missav.ws';
     const url = `${baseUrl}/${urlPath}`;
 
-    const browser = await puppeteer({ stealth: true });
+    const browser = await puppeteer();
     const page = await browser.newPage();
     await page.setRequestInterception(true);
     page.on('request', (request) => {

package/lib/routes/mit/hanlab.ts

@@ -0,0 +1,61 @@
+import { Route } from '@/types';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+
+export const route: Route = {
+    path: '/hanlab/blog',
+    categories: ['blog'],
+    example: '/mit/hanlab/blog',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['hanlab.mit.edu/blog'],
+        },
+    ],
+    name: 'HAN Lab Blog',
+    maintainers: ['johan456789'],
+    handler,
+    description: 'MIT HAN Lab pioneers research in efficient AI, advancing algorithms and hardware to make generative models faster, smarter, and more accessible.',
+};
+
+async function handler() {
+    const rootUrl = 'https://hanlab.mit.edu';
+    const currentUrl = `${rootUrl}/blog`;
+
+    const response = await got(currentUrl);
+    const $ = load(response.data);
+
+    const items = $('a.post-card-wrapper')
+        .toArray()
+        .map((item) => {
+            const el = $(item);
+            const titleEl = el.find('h3.text-title');
+            const title = titleEl.text().trim();
+            const link = new URL(el.attr('href') ?? '', rootUrl).href;
+            const description = el.find('p.text-tldr').html() ?? undefined;
+            const dateText = el.find('div.text-date').text().trim();
+            const pubDate = parseDate(dateText);
+
+            return {
+                title,
+                link,
+                description,
+                pubDate,
+            };
+        });
+
+    return {
+        title: 'MIT HAN Lab Blog',
+        link: currentUrl,
+        item: items,
+    };
+}

package/lib/routes/mit/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Massachusetts Institute of Technology',
+    url: 'mit.edu',
+};

package/lib/routes/mittrchina/index.ts

@@ -80,10 +80,10 @@ async function handler(ctx) {
                           type: article.address.split('.').pop(),
                       },
                   })
-                : (type === 'breaking'
+                : type === 'breaking'
                   ? article.content
-                  : article.summary),
-        pubDate: article.start_time ? parseDate(article.start_time, 'X') : (article.push_time ? parseDate(article.push_time, 'X') : undefined),
+                  : article.summary,
+        pubDate: article.start_time ? parseDate(article.start_time, 'X') : article.push_time ? parseDate(article.push_time, 'X') : undefined,
         id: article.id,
         link: `https://www.mittrchina.com/news/detail/${article.id}`,
     }));

package/lib/routes/mixcloud/config.ts

@@ -0,0 +1,129 @@
+export const MIXCLOUD_CONFIG = {
+    host: 'https://www.mixcloud.com',
+    imageBaseURL: 'https://thumbnailer.mixcloud.com/unsafe/480x480/',
+    graphqlURL: 'https://app.mixcloud.com/graphql',
+    decryptionKey: 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD',
+    headers: {
+        Referer: 'https://www.mixcloud.com',
+        'Content-Type': 'application/json',
+        'X-Requested-With': 'XMLHttpRequest',
+    },
+};
+
+export const TYPE_CONFIG = {
+    uploads: 'uploads',
+    reposts: 'reposted',
+    favorites: 'favorites',
+    listens: 'listeningHistory',
+    stream: 'stream',
+    playlist: 'items',
+};
+
+export const TYPE_NAMES = {
+    uploads: 'Shows',
+    reposts: 'Reposts',
+    favorites: 'Favorites',
+    listens: 'History',
+    stream: 'Stream',
+    playlist: 'Playlist',
+};
+
+export const CLOUDCAST_FIELDS = `
+  id
+  slug
+  name
+  description
+  publishDate
+  audioLength
+  picture(width: 1024, height: 1024) {
+    url
+  }
+  owner {
+    displayName
+    username
+    url
+  }
+  streamInfo {
+    url
+    hlsUrl
+    dashUrl
+  }
+  favorites {
+    totalCount
+  }
+  reposts {
+    totalCount
+  }
+  plays
+  tags {
+    tag {
+      name
+    }
+  }
+  featuringArtistList
+  isExclusive
+  restrictedReason
+  comments(first: 100) {
+    edges {
+      node {
+        comment
+        created
+        user {
+          displayName
+          username
+        }
+      }
+    }
+    totalCount
+  }
+`;
+
+export function getObjectFields(type: string): { objectType: string; objectFields: string } {
+    if (type === 'playlist') {
+        return {
+            objectType: 'playlist',
+            objectFields: `
+        name
+        description
+        picture {
+          urlRoot
+        }
+        items(first: 100) {
+          edges {
+            node {
+              cloudcast {
+                ${CLOUDCAST_FIELDS}
+              }
+            }
+          }
+          pageInfo {
+            endCursor
+            hasNextPage
+          }
+        }
+      `,
+        };
+    } else {
+        const nodeTemplate = type === 'listens' ? `node { cloudcast { ${CLOUDCAST_FIELDS} } }` : `node { ${CLOUDCAST_FIELDS} }`;
+
+        return {
+            objectType: 'user',
+            objectFields: `
+        displayName
+        biog
+        picture {
+          urlRoot
+        }
+        ${TYPE_CONFIG[type]}(first: 100) {
+          edges {
+            ${nodeTemplate}
+          }
+          pageInfo {
+            endCursor
+            hasNextPage
+          }
+        }
+      `,
+        };
+    }
+}

package/lib/routes/mixcloud/index.ts

@@ -2,14 +2,17 @@ import { Route } from '@/types';
 import got from '@/utils/got';
 import CryptoJS from 'crypto-js';
 import { parseDate } from '@/utils/parse-date';
-import { queries } from './queries';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
+import { MIXCLOUD_CONFIG, TYPE_CONFIG, TYPE_NAMES, getObjectFields } from './config';
 
 export const route: Route = {
     path: '/:username/:type?',
     categories: ['multimedia'],
     example: '/mixcloud/dholbach/uploads',
-    parameters: { username: 'Username, can be found in URL', type: 'Type, see below, uploads by default' },
+    parameters: {
+        username: 'Username, can be found in URL',
+        type: 'Type, see below, uploads by default',
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -34,123 +37,176 @@ export const route: Route = {
 | uploads | reposts | favorites | listens | stream |`,
 };
 
-async function handler(ctx) {
-    const host = 'https://www.mixcloud.com';
-    const imageBaseURL = 'https://thumbnailer.mixcloud.com/unsafe/480x480/';
-    const graphqlURL = 'https://app.mixcloud.com/graphql';
-    const headers = {
-        Referer: host,
-        'Content-Type': 'application/json',
-        'X-Requested-With': 'XMLHttpRequest',
-    };
+async function callApi(objectType: string, objectFields: string, username: string, slug?: string) {
+    const lookupKey = objectType + 'Lookup';
+    const headers = MIXCLOUD_CONFIG.headers;
+
+    const lookupParams = slug ? `, slug: "${slug}"` : '';
+    const query = `{
+    ${lookupKey}(lookup: {username: "${username}"${lookupParams}}) {
+      ${objectFields}
+    }
+  }`;
+
+    const response = await got({
+        method: 'post',
+        url: MIXCLOUD_CONFIG.graphqlURL,
+        headers,
+        json: { query },
+    });
+
+    return response.data.data[lookupKey];
+}
+
+// https://github.com/yt-dlp/yt-dlp/commits/master/yt_dlp/extractor/mixcloud.py
+function decryptXorCipher(key: string, ciphertext: string): string {
+    const decoded = CryptoJS.enc.Base64.parse(ciphertext).toString(CryptoJS.enc.Utf8);
+    return [...decoded].map((ch, i) => String.fromCodePoint((ch.codePointAt(0) || 0) ^ (key.codePointAt(i % key.length) || 0))).join('');
+}
+
+function tryDecrypt(ciphertext: string | undefined): string {
+    if (!ciphertext) {
+        return '';
+    }
+    try {
+        return decryptXorCipher(MIXCLOUD_CONFIG.decryptionKey, ciphertext);
+    } catch {
+        return ciphertext;
+    }
+}
+
+function getCloudcast(node: any, type: string): any {
+    if (type === 'playlist' || type === 'listens') {
+        return node.cloudcast;
+    }
+    return node;
+}
+
+function getPlaylistTitle(displayName: string, type: string, playlistName?: string): string {
+    if (type === 'playlist' && playlistName) {
+        return `Mixcloud - ${displayName}'s Playlist: ${playlistName}`;
+    }
+    return `Mixcloud - ${displayName}'s ${TYPE_NAMES[type] || type}`;
+}
+
+function getPlaylistLink(username: string, type: string, playlistSlug?: string): string {
+    const host = MIXCLOUD_CONFIG.host;
+    if (type === 'playlist' && playlistSlug) {
+        return `${host}/${username}/playlists/${playlistSlug}/`;
+    }
+    return `${host}/${username}/${type === 'uploads' ? '' : type + '/'}`;
+}
 
-    const type = ctx.req.param('type') ?? 'uploads';
+export async function handler(ctx) {
     const username = ctx.req.param('username');
+    const playlistSlug = ctx.req.param('playlist');
+    const type = ctx.req.param('type') ?? (playlistSlug ? 'playlist' : 'uploads');
 
-    const config = {
-        stream: { name: 'Stream', node: 'stream' },
-        uploads: { name: 'Shows', node: 'uploads' },
-        reposts: { name: 'Reposts', node: 'reposted' },
-        favorites: { name: 'Favorites', node: 'favorites' },
-        listens: { name: 'History', node: 'listeningHistory' },
-    };
-    if (!config[type]) {
+    if (!TYPE_CONFIG[type]) {
         throw new InvalidParameterError(`Invalid type: ${type}`);
     }
 
-    const payloads = {
-        stream: {
-            query: queries.stream.query,
-            variables: {
-                lookup: {
-                    username: ctx.req.param('username'),
-                },
-            },
-        },
-        uploads: {
-            query: queries.uploads.query,
-            variables: {
-                lookup: {
-                    username: ctx.req.param('username'),
-                },
-                orderBy: 'LATEST',
-                onlyAttributedTo: '',
-                hasAttributedTo: false,
-            },
-        },
-        reposts: {
-            query: queries.reposts.query,
-            variables: {
-                lookup: {
-                    username: ctx.req.param('username'),
-                },
-            },
-        },
-        favorites: {
-            query: queries.favorites.query,
-            variables: {
-                lookup: {
-                    username: ctx.req.param('username'),
-                },
-            },
-        },
-        listens: {
-            query: queries.listens.query,
-            variables: {
-                lookup: {
-                    username: ctx.req.param('username'),
-                },
-            },
-        },
-    };
+    const { objectType, objectFields } = getObjectFields(type);
+
+    const data = await callApi(objectType, objectFields, username, playlistSlug);
+
+    if (!data) {
+        throw new Error(`${type === 'playlist' ? 'Playlist' : 'User'} not found`);
+    }
 
-    const data = (
-        await got({
-            method: 'post',
-            url: graphqlURL,
-            json: payloads[type],
-            headers,
+    const isPlaylist = type === 'playlist';
+    const displayName = isPlaylist ? username : data.displayName;
+    const description = isPlaylist ? data.description : data.biog;
+    const picture = data.picture;
+    const image = picture && picture.urlRoot ? `${MIXCLOUD_CONFIG.imageBaseURL}${picture.urlRoot}` : '';
+
+    const itemsData = data[TYPE_CONFIG[type]];
+    const edges = itemsData?.edges || [];
+
+    const items = edges
+        .map((edge: any) => {
+            const cloudcast = getCloudcast(edge.node, type);
+
+            if (!cloudcast) {
+                return null;
+            }
+
+            const streamInfo = cloudcast.streamInfo || {};
+            const enclosureUrl = tryDecrypt(streamInfo.url);
+            const tags = cloudcast.tags?.map((t: any) => t.tag?.name).filter(Boolean) || [];
+
+            let richDescription = cloudcast.description?.replaceAll('\n', '<br>') || '';
+
+            if (cloudcast.featuringArtistList && cloudcast.featuringArtistList.length > 0) {
+                const artists = cloudcast.featuringArtistList.slice(0, 5).join(', ');
+                richDescription += richDescription ? '<br><br>' : '';
+                richDescription += `<strong>Featured Artists:</strong> ${artists}`;
+                if (cloudcast.featuringArtistList.length > 5) {
+                    richDescription += ` and ${cloudcast.featuringArtistList.length - 5} more...`;
+                }
+            }
+
+            const stats: string[] = [];
+            if (cloudcast.plays) {
+                stats.push(`${cloudcast.plays} plays`);
+            }
+            if (cloudcast.favorites?.totalCount) {
+                stats.push(`${cloudcast.favorites.totalCount} favorites`);
+            }
+            if (cloudcast.reposts?.totalCount) {
+                stats.push(`${cloudcast.reposts.totalCount} reposts`);
+            }
+            if (cloudcast.comments?.totalCount) {
+                stats.push(`${cloudcast.comments.totalCount} comments`);
+            }
+
+            if (stats.length > 0) {
+                richDescription += richDescription ? '<br><br>' : '';
+                richDescription += `<em>${stats.join(' • ')}</em>`;
+            }
+
+            return {
+                title: cloudcast.name,
+                author: cloudcast.owner?.displayName || username,
+                description: richDescription,
+                pubDate: parseDate(cloudcast.publishDate),
+                guid: Buffer.from(cloudcast.id, 'base64').toString(),
+                link: `${MIXCLOUD_CONFIG.host}/${cloudcast.owner?.username || username}/${cloudcast.slug}`,
+                itunes_item_image: cloudcast.picture?.url || '',
+                itunes_duration: cloudcast.audioLength,
+                enclosure_url: enclosureUrl,
+                enclosure_type: 'audio/x-m4a',
+                itunes_author: cloudcast.owner?.displayName || username,
+                category: tags,
+                comments: cloudcast.comments?.totalCount || 0,
+                upvotes: cloudcast.favorites?.totalCount || 0,
+                media: enclosureUrl
+                    ? {
+                          content: {
+                              url: enclosureUrl,
+                              type: 'audio/x-m4a',
+                              duration: cloudcast.audioLength,
+                          },
+                          thumbnail: cloudcast.picture?.url
+                              ? {
+                                    url: cloudcast.picture.url,
+                                }
+                              : undefined,
+                      }
+                    : undefined,
+            };
         })
-    ).data.data;
-
-    const biog = data.user.biog;
-    const image = `${imageBaseURL}${data.user.picture.urlRoot}`;
-
-    // https://github.com/ytdl-org/youtube-dl/blob/f1487d4fca40fd37d735753e24a7bae53a1b1513/youtube_dl/extractor/mixcloud.py#L72-L79
-    const decryptionKey = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD';
-    const decryptXorCipher = (key, cipherText) => {
-        const cipher = CryptoJS.enc.Base64.parse(cipherText);
-        const decrypted = cipher.toString(CryptoJS.enc.Utf8);
-        let result = '';
-        for (let i = 0; i < decrypted.length; i++) {
-            result += String.fromCharCode(decrypted.charCodeAt(i) ^ key.charCodeAt(i % key.length));
-        }
-        return result;
-    };
+        .filter(Boolean);
 
-    const items = data.user[config[type].node].edges.map((edge) => {
-        const item = type === 'listens' ? edge.node.cloudcast : edge.node;
-        return {
-            title: item.name,
-            author: item.owner.displayName,
-            description: item.description.replaceAll('\n', '<br>'),
-            pubDate: parseDate(item.publishDate),
-            guid: Buffer.from(item.id, 'base64').toString('utf8'),
-            link: `${host}/${username}/${item.slug}`,
-            itunes_item_image: `${imageBaseURL}${item.picture.urlRoot}`,
-            itunes_duration: item.audioLength,
-            enclosure_url: decryptXorCipher(decryptionKey, item.streamInfo.url),
-            enclosure_type: 'audio/x-m4a',
-            upvotes: item.favorites.totalCount,
-        };
-    });
+    const title = getPlaylistTitle(displayName, type, data.name);
+    const link = getPlaylistLink(username, type, playlistSlug);
 
     return {
-        title: `Mixcloud - ${data.user.displayName}'s ${config[type].name}`,
-        description: biog.replaceAll('\n', '<br>'),
-        itunes_author: data.user.displayName,
+        title,
+        description: description?.replaceAll('\n', '<br>') || '',
+        itunes_author: displayName,
         image,
-        link: `${host}/${username}/${type}/`,
+        link,
         item: items,
     };
 }

package/lib/routes/mixcloud/user-playlist.ts

@@ -0,0 +1,31 @@
+import { Route } from '@/types';
+import { handler } from './index';
+
+export const route: Route = {
+    path: '/:username/playlists/:playlist',
+    categories: ['multimedia'],
+    example: '/mixcloud/dholbach/playlists/ecclectic-dance',
+    parameters: {
+        username: 'Username, can be found in URL',
+        playlist: 'Playlist slug, can be found in URL',
+    },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: true,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['mixcloud.com/:username/playlists/:playlist'],
+        },
+        {
+            source: ['www.mixcloud.com/:username/playlists/:playlist'],
+        },
+    ],
+    name: 'Playlist',
+    maintainers: ['Misaka13514'],
+    handler,
+};

package/lib/routes/mixi2/community.ts

@@ -0,0 +1,72 @@
+import { type Data, type Route, ViewType } from '@/types';
+import type { Context } from 'hono';
+import { CONFIG_OPTIONS, generatePostDataItem, getClient, postFilter } from './utils';
+
+const handler = async (ctx: Context) => {
+    const limit = Number.parseInt(ctx.req.query('limit') ?? '20', 10);
+    const communityId = ctx.req.param('id');
+    const mediaOnly = ctx.req.param('media') === 'media';
+
+    const client = getClient();
+
+    const [communityInfo, postsData] = await Promise.all([
+        client.getCommunity({
+            communityId,
+        }),
+        client.getCommunityTimeline({
+            communityId,
+            limit,
+            mediaOnly,
+        }),
+    ]);
+
+    const personasData = await client.getPersonas({
+        personaIds: postsData?.posts.map((post) => post.personaId),
+    });
+
+    return {
+        title: `${communityInfo.community.name} - ${mediaOnly ? 'メディア' : 'ポスト'}`,
+        description: communityInfo.community.purpose.replaceAll('\n', ' '),
+        link: `https://mixi.social/communities/${communityId}/about`,
+        image: communityInfo.community.coverImage.postImage?.largeImageUrl,
+        item:
+            postsData?.posts?.filter(postFilter).map((post) => ({
+                title: communityInfo.community.name,
+                ...generatePostDataItem(post, personasData.personas),
+            })) ?? [],
+    } as Data;
+};
+
+export const route: Route = {
+    path: '/community/:id/:media?',
+    name: 'コミュニティ',
+    categories: ['social-media'],
+    example: '/mixi2/community/62e7e813-d242-4c54-a0ee-0aab5b2bbad2',
+    parameters: {
+        id: {
+            description: 'コミュニティID',
+        },
+        media: {
+            description: '`media`を入力するとメディアを含むポストのみを取得、デフォルトは空で全てのポストを取得',
+        },
+    },
+    features: {
+        supportRadar: true,
+        requireConfig: CONFIG_OPTIONS,
+    },
+    radar: [
+        {
+            source: ['mixi.social/communities/:id', 'mixi.social/communities/:id/about'],
+            target: '/community/:id',
+            title: 'コミュニティ - ポスト',
+        },
+        {
+            source: ['mixi.social/communities/:id', 'mixi.social/communities/:id/about'],
+            target: '/community/:id/media',
+            title: 'コミュニティ - メディア',
+        },
+    ],
+    view: ViewType.SocialMedia,
+    handler,
+    maintainers: ['KarasuShin'],
+};

package/lib/routes/mixi2/discovery.ts

@@ -0,0 +1,52 @@
+import { type Data, type Route, ViewType } from '@/types';
+import type { Context } from 'hono';
+import { CONFIG_OPTIONS, generatePostDataItem, getClient, postFilter } from './utils';
+
+const handler = async (ctx: Context) => {
+    const limit = Number.parseInt(ctx.req.query('limit') ?? '20', 10);
+
+    const client = getClient();
+
+    const data = await client.getRecommendedTimeline({
+        limit,
+    });
+
+    const personasData = await client.getPersonas({
+        personaIds: data?.posts?.map((post) => post.personaId) ?? [],
+    });
+
+    return {
+        title: '発見',
+        link: 'https://mixi.social/home/discovery',
+        image: 'https://mixi.social/_next/static/media/image_logo.8bb36f11.svg',
+        item:
+            data?.posts?.filter(postFilter).map((post) => {
+                const author = personasData.personas.find((persona) => persona.personaId === post.personaId);
+                return {
+                    title: `@${author?.name}`,
+                    ...generatePostDataItem(post, personasData.personas),
+                };
+            }) ?? [],
+    } as Data;
+};
+
+export const route: Route = {
+    path: '/discovery',
+    name: '発見',
+    categories: ['social-media'],
+    example: '/mixi2/discovery',
+    features: {
+        supportRadar: true,
+        requireConfig: CONFIG_OPTIONS,
+    },
+    radar: [
+        {
+            source: ['mixi.social/home/discovery'],
+            target: '/discovery',
+            title: '発見',
+        },
+    ],
+    view: ViewType.SocialMedia,
+    handler,
+    maintainers: ['KarasuShin'],
+};