rsshub 1.0.0-master.f9b85f5 → 1.0.0-master.f9c381a

package/lib/routes/telegram/channel-media.ts

@@ -0,0 +1,249 @@
+import InvalidParameterError from '@/errors/types/invalid-parameter';
+import { Route } from '@/types';
+import { Context } from 'hono';
+import { stream } from 'hono/streaming';
+import { Api, TelegramClient } from 'telegram';
+import { IterDownloadFunction } from 'telegram/client/downloads.js';
+import { getAppropriatedPartSize } from 'telegram/Utils.js';
+import { config } from '@/config';
+import cacheModule from '@/utils/cache/index';
+import { getClient, getDocument, getFilename, unwrapMedia } from './tglib/client';
+import { returnBigInt as bigInt } from 'telegram/Helpers.js';
+
+/**
+ * https://core.telegram.org/api/files#stripped-thumbnails
+ * @param bytes Buffer
+ * @returns Buffer jpeg
+ */
+function ExpandInlineBytes(bytes: Buffer) {
+    if (bytes.length < 3 || bytes[0] !== 0x1) {
+        throw new Error('cannot inflate a stripped jpeg');
+    }
+    const header = Buffer.from([
+        0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xFF, 0xDB, 0x00, 0x43, 0x00, 0x28, 0x1C, 0x1E, 0x23, 0x1E, 0x19, 0x28, 0x23, 0x21, 0x23, 0x2D, 0x2B,
+        0x28, 0x30, 0x3C, 0x64, 0x41, 0x3C, 0x37, 0x37, 0x3C, 0x7B, 0x58, 0x5D, 0x49, 0x64, 0x91, 0x80, 0x99, 0x96, 0x8F, 0x80, 0x8C, 0x8A, 0xA0, 0xB4, 0xE6, 0xC3, 0xA0, 0xAA, 0xDA, 0xAD, 0x8A, 0x8C, 0xC8, 0xFF, 0xCB, 0xDA, 0xEE,
+        0xF5, 0xFF, 0xFF, 0xFF, 0x9B, 0xC1, 0xFF, 0xFF, 0xFF, 0xFA, 0xFF, 0xE6, 0xFD, 0xFF, 0xF8, 0xFF, 0xDB, 0x00, 0x43, 0x01, 0x2B, 0x2D, 0x2D, 0x3C, 0x35, 0x3C, 0x76, 0x41, 0x41, 0x76, 0xF8, 0xA5, 0x8C, 0xA5, 0xF8, 0xF8, 0xF8,
+        0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8,
+        0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xF8, 0xFF, 0xC0, 0x00, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, 0xFF, 0xC4, 0x00, 0x1F, 0x00, 0x00, 0x01, 0x05,
+        0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0xC4, 0x00, 0xB5, 0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04,
+        0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7D, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xA1, 0x08, 0x23, 0x42, 0xB1, 0xC1,
+        0x15, 0x52, 0xD1, 0xF0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0A, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A,
+        0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96,
+        0x97, 0x98, 0x99, 0x9A, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
+        0xD8, 0xD9, 0xDA, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFF, 0xC4, 0x00, 0x1F, 0x01, 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+        0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0xC4, 0x00, 0xB5, 0x11, 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, 0x07, 0x05, 0x04, 0x04, 0x00,
+        0x01, 0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xA1, 0xB1, 0xC1, 0x09, 0x23, 0x33, 0x52, 0xF0, 0x15, 0x62,
+        0x72, 0xD1, 0x0A, 0x16, 0x24, 0x34, 0xE1, 0x25, 0xF1, 0x17, 0x18, 0x19, 0x1A, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x53, 0x54, 0x55, 0x56, 0x57,
+        0x58, 0x59, 0x5A, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A,
+        0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xE2,
+        0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFF, 0xDA, 0x00, 0x0C, 0x03, 0x01, 0x00, 0x02, 0x11, 0x03, 0x11, 0x00, 0x3F, 0x00,
+    ]);
+    const footer = Buffer.from([0xFF, 0xD9]);
+    const real = Buffer.alloc(header.length + bytes.length + footer.length);
+    header.copy(real);
+    bytes.copy(real, header.length, 3);
+    bytes.copy(real, 164, 1, 2);
+    bytes.copy(real, 166, 2, 3);
+    footer.copy(real, header.length + bytes.length, 0);
+    return real;
+}
+
+function sortThumb(thumb: Api.TypePhotoSize) {
+    if (thumb instanceof Api.PhotoStrippedSize) {
+        return thumb.bytes.length;
+    }
+    if (thumb instanceof Api.PhotoCachedSize) {
+        return thumb.bytes.length;
+    }
+    if (thumb instanceof Api.PhotoSize) {
+        return thumb.size;
+    }
+    if (thumb instanceof Api.PhotoSizeProgressive) {
+        return Math.max(...thumb.sizes);
+    }
+    return 0;
+}
+
+function chooseLargestThumb(thumbs: Api.TypePhotoSize[]) {
+    thumbs = [...thumbs].sort((a, b) => sortThumb(a) - sortThumb(b));
+    return thumbs.pop();
+}
+
+export async function* streamThumbnail(client: TelegramClient, doc: Api.Document) {
+    if (doc.thumbs?.length ?? 0 > 0) {
+        const size = chooseLargestThumb(doc.thumbs!);
+        if (size instanceof Api.PhotoCachedSize || size instanceof Api.PhotoStrippedSize) {
+            yield ExpandInlineBytes(size.bytes);
+        } else {
+            yield* streamDocument(client, doc, size && 'type' in size ? size.type : '');
+        }
+        return;
+    }
+    throw new Error('no thumbnails available');
+}
+
+export async function* streamDocument(client: TelegramClient, obj: Api.Document, thumbSize = '', offset?: bigInt.BigInteger, limit?: bigInt.BigInteger) {
+    const chunkSize = (obj.size ? getAppropriatedPartSize(obj.size) : 64) * 1024;
+    const iterFileParams: IterDownloadFunction = {
+        file: new Api.InputDocumentFileLocation({
+            id: obj.id,
+            accessHash: obj.accessHash,
+            fileReference: obj.fileReference,
+            thumbSize,
+        }),
+        chunkSize,
+        requestSize: 512 * 1024, // MAX_CHUNK_SIZE
+        dcId: obj.dcId,
+        offset: undefined,
+        limit: undefined,
+    };
+    if (offset) {
+        iterFileParams.offset = offset;
+    }
+    if (limit) {
+        iterFileParams.limit = limit.valueOf();
+    }
+    // console.log('starting iterDownload');
+    const stream = client.iterDownload(iterFileParams);
+    yield* stream;
+    await stream.close();
+}
+
+function parseRange(range: string, length: bigInt.BigInteger) {
+    if (!range) {
+        return [];
+    }
+    const [typ, segstr] = range.split('=');
+    if (typ !== 'bytes') {
+        throw new InvalidParameterError(`unsupported range: ${typ}`);
+    }
+    const segs = segstr.split(',').map((s) => s.trim());
+    const parsedSegs: bigInt.BigInteger[][] = [];
+    for (const seg of segs) {
+        const range = seg
+            .split('-', 2)
+            .filter((v) => !!v)
+            .map((v) => bigInt(v));
+        if (range.length < 2) {
+            if (seg.startsWith('-')) {
+                range.unshift(bigInt(0));
+            } else {
+                range.push(length.subtract(bigInt(1)));
+            }
+        }
+        parsedSegs.push(range);
+    }
+    return parsedSegs;
+}
+
+export async function configureMiddlewares(ctx: Context) {
+    // media is too heavy to cache in memory or redis, and lock-up is not needed
+    await cacheModule.set(ctx.get('cacheControlKey'), '0', config.cache.requestTimeout);
+    ctx.req.raw.headers.delete('Accept-Encoding'); // avoid hono compress() middleware detecting Accept-Encoding on req
+}
+
+function streamResponse(c: Context, bodyIter: AsyncGenerator<Buffer>) {
+    return stream(c, async (stream) => {
+        let aborted = false;
+        stream.onAbort(() => {
+            // console.log(`stream aborted`);
+            aborted = true;
+        });
+        for await (const chunk of bodyIter) {
+            if (aborted) {
+                break;
+            }
+            // console.log(`writing ${chunk.length / 1024}kB`);
+            await stream.write(chunk);
+        }
+        // console.log(`done streamResponse`);
+    });
+}
+
+export const route: Route = {
+    path: '/media/:entityName/:messageId',
+    categories: ['social-media'],
+    example: '/telegram/media/telegram/1233',
+    parameters: { entityName: 'entity name', messageId: 'message id' },
+    features: {
+        requireConfig: [
+            {
+                name: 'TELEGRAM_SESSION',
+                optional: false,
+                description: 'Telegram API Authentication',
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [],
+    name: 'Channel Media',
+    maintainers: ['synchrone'],
+    handler,
+    description: `
+::: tip
+  Serves telegram media like pictures, video or files.
+:::
+`,
+};
+
+export async function handleMedia(media: Api.TypeMessageMedia, client: TelegramClient, ctx: Context) {
+    if (media instanceof Api.MessageMediaPhoto) {
+        const buf = await client.downloadMedia(media);
+        return new Response(buf, { headers: { 'Content-Type': 'image/jpeg' } });
+    }
+
+    const doc = getDocument(media);
+    if (doc) {
+        if ('thumb' in ctx.req.query()) {
+            ctx.header('Content-Type', 'image/jpeg');
+            return streamResponse(ctx, streamThumbnail(client, doc));
+        }
+        ctx.header('Content-Type', doc.mimeType);
+        ctx.header('Accept-Ranges', 'bytes');
+        ctx.header('Content-Security-Policy', "default-src 'self'; script-src 'none'");
+
+        const rangeHeader = ctx.req.header('Range') ?? '';
+        const range = parseRange(rangeHeader, doc.size);
+        if (range.length > 1) {
+            return ctx.text('Not Satisfiable', 416);
+        }
+
+        if (range.length === 0) {
+            ctx.header('Content-Length', doc.size.toString());
+            if (!doc.mimeType.startsWith('video/') && !doc.mimeType.startsWith('audio/') && !doc.mimeType.startsWith('image/')) {
+                ctx.header('Content-Disposition', `attachment; filename="${encodeURIComponent(getFilename(media))}"`);
+            }
+            return streamResponse(ctx, streamDocument(client, doc));
+        } else {
+            const [offset, limit] = range[0];
+            // console.log(`Range: ${rangeHeader}`);
+            ctx.status(206); // partial content
+            ctx.header('Content-Length', limit.subtract(offset).add(1).toString());
+            ctx.header('Content-Range', `bytes ${offset}-${limit}/${doc.size}`);
+            return streamResponse(ctx, streamDocument(client, doc, '', offset, limit));
+        }
+    }
+
+    return ctx.text(media.className, 415);
+}
+
+export default async function handler(ctx: Context) {
+    await configureMiddlewares(ctx);
+    const client = await getClient();
+
+    const { entityName, messageId } = ctx.req.param();
+    const entity = await client.getInputEntity(entityName);
+    const msgs = await client.getMessages(entity, {
+        ids: [Number(messageId)],
+    });
+    const media = await unwrapMedia(msgs[0]?.media);
+    if (!media) {
+        return ctx.text('Unknown media', 404);
+    }
+
+    return await handleMedia(media, client, ctx);
+}

package/lib/routes/telegram/channel.ts

@@ -148,6 +148,11 @@ For backward compatibility reasons, invalid \`routeParams\` will be treated as \
 };
 
 async function handler(ctx) {
+    const useWeb = ctx.req.param('routeParams') || !config.telegram.session;
+    if (!useWeb) {
+        return tglibchannel(ctx);
+    }
+
     const username = ctx.req.param('username');
     let routeParams = ctx.req.param('routeParams');
     let showLinkPreview = true;
@@ -219,10 +224,6 @@ async function handler(ctx) {
         : $('.tgme_widget_message_wrap:not(.tgme_widget_message_wrap:has(.service_message,.tme_no_messages_found))'); // also exclude service messages
 
     if (list.length === 0 && $('.tgme_channel_history').length === 0) {
-        if (config.telegram.session) {
-            return tglibchannel(ctx);
-        }
-
         throw new Error(`Unable to fetch message feed from this channel. Please check this URL to see if you can view the message preview: ${resourceUrl}`);
     }
 

package/lib/routes/telegram/stories.ts

@@ -0,0 +1,130 @@
+/* eslint-disable no-await-in-loop */
+import NotFoundError from '@/errors/types/not-found';
+import { configureMiddlewares, handleMedia } from '@/routes/telegram/channel-media';
+import { Data, DataItem, Route } from '@/types';
+import { Context } from 'hono';
+import { Api } from 'telegram';
+import { getClient, getStory, unwrapMedia } from './tglib/client';
+import { getGeoLink, getMediaLink } from './tglib/channel';
+
+export const route: Route = {
+    path: '/stories/:username/:story?',
+    categories: ['social-media'],
+    example: '/telegram/stories/telegram',
+    parameters: { username: 'entity name', story: 'story' },
+    features: {
+        requireConfig: [
+            {
+                name: 'TELEGRAM_SESSION',
+                optional: false,
+                description: 'Telegram API Authentication',
+            },
+            {
+                name: 'TELEGRAM_API_ID',
+                optional: true,
+                description: 'Telegram API ID',
+            },
+            {
+                name: 'TELEGRAM_API_HASH',
+                optional: true,
+                description: 'Telegram API Hash',
+            },
+            {
+                name: 'TELEGRAM_MAX_CONCURRENT_DOWNLOADS',
+                optional: true,
+                description: 'Telegram Max Concurrent Downloads',
+            },
+            {
+                name: 'TELEGRAM_PROXY_HOST',
+                optional: true,
+                description: 'Telegram Proxy Host',
+            },
+            {
+                name: 'TELEGRAM_PROXY_PORT',
+                optional: true,
+                description: 'Telegram Proxy Port',
+            },
+            {
+                name: 'TELEGRAM_PROXY_SECRET',
+                optional: true,
+                description: 'Telegram Proxy Secret',
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [],
+    name: 'Stories',
+    maintainers: ['synchrone'],
+    handler,
+    description: ``,
+};
+
+function getMediaAreas(mediaAreas?: Api.TypeMediaArea[]) {
+    let description = '';
+    for (const area of mediaAreas ?? []) {
+        if (area instanceof Api.MediaAreaChannelPost) {
+            // TODO: fetch area.msgId and display inline
+        } else if ((area instanceof Api.MediaAreaGeoPoint || area instanceof Api.MediaAreaVenue) && area.geo instanceof Api.GeoPoint) {
+            description += getGeoLink(area.geo);
+        } else if (area instanceof Api.MediaAreaSuggestedReaction) {
+            if (area.reaction instanceof Api.ReactionEmoji) {
+                description += area.reaction.emoticon;
+            } else if (area.reaction instanceof Api.ReactionCustomEmoji) {
+                // TODO: fetch area.reaction.documentId and display inline
+            }
+        }
+    }
+    return description;
+}
+
+export default async function handler(ctx: Context) {
+    const c = await getClient();
+    const { username, story } = ctx.req.param();
+    if (!username) {
+        throw new NotFoundError();
+    }
+    const peer = await c.getInputEntity(username);
+    if (story) {
+        const storyItem = await getStory(peer, Number(story));
+        await configureMiddlewares(ctx);
+        return await handleMedia(storyItem.media, c, ctx);
+    }
+
+    const storiesRes = await c.invoke(new Api.stories.GetPeerStories({ peer }));
+
+    const item: DataItem[] = [];
+    for (const story of storiesRes.stories.stories) {
+        if (!(story instanceof Api.StoryItem)) {
+            // story is deleted (archived) or skipped
+            continue;
+        }
+        const src = `${new URL(ctx.req.url).origin}/telegram/stories/${username}/${story.id}`;
+        const pubDate = new Date(story.date * 1000).toUTCString();
+        const media = await unwrapMedia(story.media);
+        if (!media) {
+            // cannot load the story
+            continue;
+        }
+
+        const description = getMediaLink(src, media) + getMediaAreas(story.mediaAreas);
+        item.push({
+            title: story.caption ?? pubDate,
+            description,
+            pubDate,
+            link: `https://t.me/${username}/s/${story.id}`,
+            author: username,
+        });
+    }
+
+    return {
+        title: `Stories of @${username}`,
+        link: `https://t.me/${username}`,
+        item,
+        allowEmpty: ctx.req.param('id') === 'allow_empty',
+        description: `Stories of @${username} on Telegram`,
+    } as Data;
+}