telegram-pm 0.1.0__py3-none-any.whl

telegram_pm/config.py

@@ -0,0 +1,25 @@
+from os import environ
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+class BaseConfig: ...
+
+
+class HttpClientConfig(BaseConfig):
+    retries: int = int(environ.get("HTTP_RETRIES", 3))
+    backoff: int = int(environ.get("HTTP_BACKOFF", 3))
+    timeout: int = int(environ.get("HTTP_TIMEOUT", 30))
+
+
+class TelegramConfig(BaseConfig):
+    base_url: str = environ.get("TELEGRAM_BASE_URL", "https://t.me")
+
+    before_param_size: int = int(environ.get("TELEGRAM_BEFORE_PARAM_SIZE", 20))
+    iteration_in_preview_count: int = int(environ.get("TELEGRAM_PARSE_REPEAT_COUNT", 5))
+    sleep_time_seconds: int = int(environ.get("TELEGRAM_SLEEP_TIME_SECONDS", 60))
+    sleep_after_error_request: int = int(
+        environ.get("TELEGRAM_SLEEP_AFTER_ERROR_REQUEST", 30)
+    )

telegram_pm/database/db.py

@@ -0,0 +1,143 @@
+import json
+from typing import List
+from dataclasses import asdict
+from contextlib import asynccontextmanager
+
+import aiosqlite
+
+from telegram_pm.entities import Post
+
+
+class DatabaseProcessor:
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+        self._pool = None
+
+    async def initialize(self):
+        async with self._get_connection() as conn:
+            await conn.execute("PRAGMA journal_mode=WAL")
+            await conn.execute("PRAGMA synchronous=NORMAL")
+            await conn.execute("PRAGMA cache_size=-10000")  # 10MB кэша
+            await conn.execute("PRAGMA temp_store=MEMORY")
+            await conn.commit()
+
+    @asynccontextmanager
+    async def _get_connection(self):
+        conn = await aiosqlite.connect(self.db_path, timeout=30, isolation_level=None)
+        conn.row_factory = aiosqlite.Row
+        try:
+            yield conn
+        finally:
+            await conn.close()
+
+    @asynccontextmanager
+    async def _get_cursor(self):
+        async with self._get_connection() as conn:
+            cursor = await conn.cursor()
+            try:
+                yield cursor
+                await conn.commit()
+            except Exception as e:
+                await conn.rollback()
+                raise e
+
+    async def table_exists(self, table_name: str) -> bool:
+        async with self._get_cursor() as cursor:
+            await cursor.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+                (table_name,),
+            )
+            return await cursor.fetchone() is not None
+
+    async def create_table_from_post(self, table_name: str):
+        columns = [
+            "url TEXT PRIMARY KEY",
+            "username TEXT",
+            "id INTEGER",
+            "date TEXT NOT NULL",
+            "text TEXT",
+            "replied_post_url TEXT",
+            "urls TEXT",  # JSON
+            "url_preview TEXT",
+            "photo_urls TEXT",  # JSON
+            "video_urls TEXT",  # JSON
+            "round_video_url TEXT",
+            "files TEXT",  # JSON
+            "tags TEXT",  # JSON
+            "created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP",
+            "forwarded_from_url TEXT",
+            "forwarded_from_name TEXT",
+        ]
+
+        async with self._get_cursor() as cursor:
+            await cursor.execute(
+                f"""
+                CREATE TABLE IF NOT EXISTS {table_name} (
+                    {", ".join(columns)}
+                )
+                """
+            )
+            await cursor.execute(
+                f"CREATE INDEX IF NOT EXISTS idx_{table_name}_date ON {table_name}(date)"
+            )
+
+    async def insert_posts_batch(self, table_name: str, posts: List[Post]):
+        if not posts:
+            return
+
+        columns = [
+            "url",
+            "username",
+            "id",
+            "date",
+            "text",
+            "replied_post_url",
+            "urls",
+            "url_preview",
+            "photo_urls",
+            "video_urls",
+            "round_video_url",
+            "files",
+            "tags",
+            "forwarded_from_url",
+            "forwarded_from_name",
+        ]
+
+        placeholders = ", ".join(["?"] * len(columns))
+        query = f"""
+            INSERT OR IGNORE INTO {table_name}
+            ({", ".join(columns)})
+            VALUES ({placeholders})
+        """
+
+        async with self._get_cursor() as cursor:
+            data = []
+            for post in posts:
+                post_dict = asdict(post)
+                for field in ["urls", "photo_urls", "video_urls", "files", "tags"]:
+                    post_dict[field] = json.dumps(post_dict[field])
+                data.append(tuple(post_dict[col] for col in columns))
+            await cursor.executemany(query, data)
+
+    async def is_table_empty(self, table_name: str) -> bool:
+        async with self._get_cursor() as cursor:
+            await cursor.execute(f"SELECT 1 FROM {table_name} LIMIT 1")
+            return await cursor.fetchone() is None
+
+    async def drop_table_if_empty(self, table_name: str):
+        if await self.table_exists(table_name) and await self.is_table_empty(
+            table_name
+        ):
+            async with self._get_cursor() as cursor:
+                await cursor.execute(f"DROP TABLE {table_name}")
+
+    async def post_exists(self, table_name: str, url: str) -> bool:
+        query = f"SELECT 1 FROM {table_name} WHERE url = ? LIMIT 1"
+
+        async with self._get_cursor() as cursor:
+            await cursor.execute(query, (url,))
+            return await cursor.fetchone() is not None
+
+    async def close(self):
+        if hasattr(self, "conn") and self.conn:
+            await self.conn.close()

telegram_pm/entities.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Post:
+    username: str
+    url: str
+    date: str
+    id: int | None
+    text: str | None = None
+    replied_post_url: str | None = None
+    urls: list[str] = field(default_factory=list)
+    url_preview: str | None = None
+    photo_urls: list[str] = field(default_factory=list[str])
+    video_urls: list[str] = field(default_factory=list[str])
+    round_video_url: str | None = None
+    files: list[dict[str, str]] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list[str])
+    forwarded_from_url: str | None = None
+    forwarded_from_name: str | None = None

telegram_pm/http_client/client.py

@@ -0,0 +1,22 @@
+import httpx
+from retry import retry
+
+from telegram_pm.utils.logger import logger
+from telegram_pm.config import HttpClientConfig
+
+
+class HttpClient:
+    def __init__(self):
+        self.client = httpx.AsyncClient(
+            transport=httpx.AsyncHTTPTransport(
+                verify=False,
+                retries=HttpClientConfig.retries,
+            ),
+            timeout=HttpClientConfig.timeout,
+            verify=False,
+        )
+
+    @retry(backoff=HttpClientConfig.backoff, logger=logger)  # type: ignore[arg-type]
+    async def request(self, url: str, method: str = "GET", **kwargs) -> httpx.Response:
+        response = await self.client.request(method=method, url=url, **kwargs)
+        return response

telegram_pm/parsers/base.py

@@ -0,0 +1 @@
+class BaseParser: ...

telegram_pm/parsers/post.py

@@ -0,0 +1,280 @@
+import typer
+from bs4 import BeautifulSoup, PageElement
+
+from telegram_pm import utils
+from telegram_pm.entities import Post
+from telegram_pm.utils.logger import logger
+from telegram_pm.parsers.base import BaseParser
+from telegram_pm.config import TelegramConfig
+from telegram_pm.parsers.tag_options import PostParseConfig, TagOptions
+
+
+class PostsParser(BaseParser):
+    """
+    Posts parsers from preview page
+    """
+
+    def __init__(self, verbose: bool = False):
+        self.__verbose: bool = verbose
+        self._tag_ops = PostParseConfig
+
+    @staticmethod
+    def get_post_attribute(
+        post: PageElement,
+        tab_ops: TagOptions,
+        extract_field: str,
+        _warn_log_enable: bool = True,
+    ) -> str | None:
+        post_attribute = post.find(name=tab_ops.tag, attrs=tab_ops.attrs)  # type: ignore[attr-defined]
+        if post_attribute:
+            if extract_field == "text":
+                return post_attribute.text
+            return post_attribute.get(extract_field)
+        if _warn_log_enable:
+            logger.warning(f"Not found. '{tab_ops.tag}': '{tab_ops.attrs}'")
+        return None
+
+    def get_channel_id(self, post: PageElement) -> int | None:
+        channel_base64 = self.get_post_attribute(
+            post=post,
+            tab_ops=self._tag_ops.channel_id,
+            extract_field="data-view",
+            _warn_log_enable=False,
+        )
+        if not channel_base64:
+            return None
+        channel_id = utils.parse.decode_channel_id(channel_base64)
+        return channel_id
+
+    @staticmethod
+    def get_urls_from_styles(post: PageElement, tag_pos: TagOptions) -> list[str]:
+        urls = []
+        styles_list = post.find_all(name=tag_pos.tag, attrs=tag_pos.attrs)  # type: ignore[attr-defined]
+        for style in styles_list:
+            urls.append(
+                utils.parse.extract_url_from_style(style_content=style.get("style", ""))
+            )
+        return urls  # type: ignore[return-value]
+
+    def get_posts(self, bs_preview_content: BeautifulSoup) -> list[PageElement]:
+        posts_list = utils.parse.extract_element(
+            bs_content=bs_preview_content,
+            tag_ops=self._tag_ops.post_block,
+        )
+        return posts_list
+
+    def get_post_url(self, username: str, post: PageElement) -> str:
+        post_url = self.get_post_attribute(
+            post=post,
+            tab_ops=self._tag_ops.post_url,
+            extract_field="href",
+        )
+        if post_url.startswith(f"{TelegramConfig.base_url}/"):  # type: ignore[union-attr]
+            post_url = post_url.split("/")[-1]  # type: ignore[union-attr]
+            post_url = f"{TelegramConfig.base_url}/{username}/{post_url}"
+        return post_url  # type: ignore[return-value]
+
+    def get_post_date(self, post: PageElement) -> str:
+        return self.get_post_attribute(  # type: ignore[return-value]
+            post=post,
+            tab_ops=self._tag_ops.date,
+            extract_field="datetime",
+        )
+
+    def get_replied_url(self, post: PageElement) -> str | None:
+        return self.get_post_attribute(
+            post=post,
+            tab_ops=self._tag_ops.replied_url,
+            extract_field="href",
+            _warn_log_enable=False,
+        )
+
+    def get_forwarded_from_url(self, post: PageElement) -> str | None:
+        return self.get_post_attribute(
+            post=post,
+            tab_ops=self._tag_ops.forwarded_from_url,
+            extract_field="href",
+            _warn_log_enable=False,
+        )
+
+    def get_forwarded_from_name(self, post: PageElement) -> str | None:
+        return self.get_post_attribute(
+            post=post,
+            tab_ops=self._tag_ops.forwarded_from_name,
+            extract_field="text",
+            _warn_log_enable=False,
+        )
+
+    def get_text(self, post: PageElement) -> str | None:
+        return self.get_post_attribute(
+            post=post,
+            tab_ops=self._tag_ops.text,
+            extract_field="text",
+            _warn_log_enable=False,
+        )
+
+    def get_photo_urls(self, post: PageElement) -> list[str]:
+        return self.get_urls_from_styles(
+            post=post,
+            tag_pos=self._tag_ops.photo_url,
+        )
+
+    def get_video_urls(self, post: PageElement) -> list[str]:
+        return self.get_urls_from_styles(
+            post=post,
+            tag_pos=self._tag_ops.video_url,
+        )
+
+    def get_round_video(self, post: PageElement) -> str | None:
+        return self.get_post_attribute(
+            post=post,
+            tab_ops=self._tag_ops.round_video_url,
+            extract_field="src",
+            _warn_log_enable=False,
+        )
+
+    def get_urls(self, post: PageElement) -> list[str]:
+        urls = set()
+        url_elements = post.find_all(  # type: ignore[attr-defined]
+            name=self._tag_ops.url.tag,
+            attrs=self._tag_ops.url.attrs,
+        )
+        for url in url_elements:
+            urls.add(url.get("href"))
+        return list(urls)
+
+    def get_url_preview(self, post: PageElement) -> str | None:
+        return self.get_post_attribute(
+            post=post,
+            tab_ops=self._tag_ops.url_preview,
+            extract_field="text",
+            _warn_log_enable=False,
+        )
+
+    def get_files(self, post: PageElement) -> list[dict[str, str]]:
+        files: list = []
+        files_elements = post.find_all(  # type: ignore[attr-defined]
+            name=self._tag_ops.file.tag,
+            attrs=self._tag_ops.file.attrs,
+        )
+        file: PageElement
+        for file in files_elements:
+            title = file.text
+            extra = file.find_next_sibling(  # type: ignore[union-attr]
+                name=self._tag_ops.file_extra.tag,
+                attrs=self._tag_ops.file_extra.attrs,
+            ).text
+            files.append({"title": title, "extra": extra})
+        return files
+
+    def get_tags(self, post: PageElement) -> list[str]:
+        tags_elements = post.find_all(  # type: ignore[attr-defined]
+            name=self._tag_ops.tag.tag,
+            attrs=self._tag_ops.tag.attrs,
+        )
+        return [tag.text for tag in tags_elements]
+
+    def parse(self, username: str, bs_preview_content: BeautifulSoup) -> list[Post]:
+        parse_results = []
+        posts_list = self.get_posts(bs_preview_content=bs_preview_content)
+        for post_element in posts_list:
+            post = Post(
+                username=username,
+                id=self.get_channel_id(post_element),
+                url=self.get_post_url(username, post_element),
+                date=self.get_post_date(post_element),
+                replied_post_url=self.get_replied_url(post_element),
+                text=self.get_text(post_element),
+                photo_urls=self.get_photo_urls(post_element),
+                video_urls=self.get_video_urls(post_element),
+                round_video_url=self.get_round_video(post_element),
+                urls=self.get_urls(post_element),
+                url_preview=self.get_url_preview(post_element),
+                files=self.get_files(post_element),
+                tags=self.get_tags(post_element),
+                forwarded_from_url=self.get_forwarded_from_url(post_element),
+                forwarded_from_name=self.get_forwarded_from_name(post_element),
+            )
+            parse_results.append(post)
+            if self.__verbose:
+                self._print_post(post=post)
+        return parse_results
+
+    @staticmethod
+    def _print_post(post: Post):
+        typer.echo("\n" + typer.style("═" * 50, fg=typer.colors.BRIGHT_MAGENTA))
+        typer.echo(
+            typer.style("🎯 Username: ", fg=typer.colors.BRIGHT_RED)
+            + typer.style(post.username, fg=typer.colors.RED)
+        )
+        typer.echo(
+            typer.style("📅 Date: ", fg=typer.colors.BRIGHT_CYAN)
+            + typer.style(post.date, fg=typer.colors.WHITE)
+        )
+
+        typer.echo(
+            typer.style("🔗 URL: ", fg=typer.colors.BRIGHT_CYAN)
+            + typer.style(post.url, fg=typer.colors.BRIGHT_BLUE, underline=True)
+        )
+
+        if post.replied_post_url:
+            typer.echo(
+                typer.style("↩️ Replied: ", fg=typer.colors.BRIGHT_YELLOW)
+                + typer.style(post.replied_post_url, fg=typer.colors.BLUE)
+            )
+
+        if post.text:
+            typer.echo("\n💬💬💬")
+            typer.echo(typer.style(post.text[:50], fg=typer.colors.GREEN))
+            typer.echo("💬💬💬")
+
+        if post.photo_urls:
+            typer.echo("\n" + typer.style("📷 Photo: ", fg=typer.colors.BRIGHT_RED))
+            for photo in post.photo_urls:
+                typer.echo(typer.style(f"  → {photo}", fg=typer.colors.RED))
+
+        if post.video_urls:
+            typer.echo("\n" + typer.style("🎥 Video: ", fg=typer.colors.BRIGHT_RED))
+            for video in post.video_urls:
+                typer.echo(typer.style(f"  → {video}", fg=typer.colors.RED))
+
+        if post.urls:
+            typer.echo("\n" + typer.style("🌐 URLs: ", fg=typer.colors.BRIGHT_MAGENTA))
+            for url in post.urls:
+                typer.echo(typer.style(f"  → {url}", fg=typer.colors.MAGENTA))
+
+        if post.url_preview:
+            typer.echo("\n👀👀👀")
+            typer.echo(
+                typer.style(
+                    f"🔍 URL preview: {post.url_preview[:50]}", fg=typer.colors.GREEN
+                )
+            )
+            typer.echo("👀👀👀")
+
+        if post.round_video_url:
+            typer.echo(
+                "\n"
+                + typer.style(
+                    f"🔍 Round video: {post.round_video_url}", fg=typer.colors.BLUE
+                )
+            )
+
+        if post.tags:
+            typer.echo(
+                "\n"
+                + typer.style("⌗ Tags: ", fg=typer.colors.BRIGHT_GREEN)
+                + typer.style(", ".join(post.tags), fg=typer.colors.GREEN)
+            )
+
+        if post.files:
+            typer.echo("\n" + typer.style("📂 Files: ", fg=typer.colors.BRIGHT_YELLOW))
+            for file in post.files:
+                print_file = file.get("title")
+                if print_file:
+                    extra = file.get("extra")
+                    if extra:
+                        print_file = f"{print_file} ({extra})"
+                typer.echo(typer.style(f"  → {print_file}", fg=typer.colors.YELLOW))
+
+        typer.echo(typer.style("═" * 50, fg=typer.colors.BRIGHT_MAGENTA) + "\n")

telegram_pm/parsers/preview.py

@@ -0,0 +1,165 @@
+import asyncio
+
+import httpx
+from bs4 import BeautifulSoup
+from structlog.contextvars import bound_contextvars
+
+from telegram_pm import utils
+from telegram_pm.entities import Post
+from telegram_pm.utils.logger import logger
+from telegram_pm.config import TelegramConfig
+from telegram_pm.parsers.base import BaseParser
+from telegram_pm.parsers.post import PostsParser
+from telegram_pm.http_client.client import HttpClient
+from telegram_pm.database.db import DatabaseProcessor
+
+
+class PreviewParser(BaseParser):
+    """
+    Telegram preview page parser
+    """
+
+    def __init__(
+        self,
+        channels: list[str],
+        db_path: str,
+        verbose: bool = False,
+    ):
+        self.channels: list[str] = channels
+        self.http_client = HttpClient()
+        self.post_parser = PostsParser(verbose=verbose)
+        self.db = DatabaseProcessor(db_path=db_path)
+        self._db_initialized = False
+        self.verbose = verbose
+
+    @staticmethod
+    def __forbidden_parse_preview(response: httpx.Response) -> bool:
+        """
+        Check parsing availability
+        :param response: httpx.Response
+        :return: bool. If True, then you can't parse preview page
+        """
+        if response.status_code in (302,):
+            return True
+        return False
+
+    @staticmethod
+    def __parse_before_param_value(post_url: str) -> int:
+        before_value = post_url.split("/")[-1]
+        return int(before_value)
+
+    async def _get_preview_page(self, preview_url: str) -> httpx.Response:
+        """
+        Get preview page
+        :param preview_url: str. Full preview URL
+        :return: httpx.Response
+        """
+        response_preview_url = await self.http_client.request(
+            url=preview_url,
+        )
+        return response_preview_url
+
+    def _parse_posts_in_preview(
+        self, username: str, response: httpx.Response
+    ) -> list[Post]:
+        bs_content = BeautifulSoup(response.text, "html5lib")
+        posts = self.post_parser.parse(username=username, bs_preview_content=bs_content)
+        return posts
+
+    async def initialize(self):
+        """Initialize database"""
+        if not self._db_initialized:
+            await self.db.initialize()
+            self._db_initialized = True
+
+    async def close(self):
+        """Clean up resources"""
+        if hasattr(self.db, "close"):
+            await self.db.close()
+
+    async def parse_channel(self, channel_username: str):
+        """Parse single channel"""
+        channel_username = utils.url.get_username_from_tg_url(channel_username)
+        with bound_contextvars(username=channel_username):
+            if not await self.db.table_exists(channel_username):
+                await self.db.create_table_from_post(channel_username)
+                await logger.ainfo("Created new table for channel")
+
+            preview_url = utils.url.build_preview_url(username=channel_username)
+            posts_result = []
+            should_break = False
+
+            for parse_repeat in range(TelegramConfig.iteration_in_preview_count):
+                if should_break:
+                    await logger.ainfo("No new posts yet")
+                    break
+
+                try:
+                    response = await self._get_preview_page(preview_url=preview_url)
+                    if not response:
+                        await logger.awarning("Can not get preview page")
+                        await asyncio.sleep(TelegramConfig.sleep_after_error_request)
+                        continue
+
+                    if self.__forbidden_parse_preview(response=response):
+                        await logger.awarning("Forbidden parsing preview")
+                        break
+
+                    parsed_posts = self._parse_posts_in_preview(
+                        username=channel_username, response=response
+                    )
+                    if not parsed_posts:
+                        await logger.awarning("No posts parsed from preview page")  # type: ignore
+                        await self.db.drop_table_if_empty(channel_username)
+                        await asyncio.sleep(TelegramConfig.sleep_after_error_request)
+                        break
+
+                    first_post_exists = await self.db.post_exists(
+                        channel_username, parsed_posts[0].url
+                    )
+                    if first_post_exists:
+                        should_break = True
+                        continue
+
+                    await self.db.insert_posts_batch(channel_username, parsed_posts)
+                    posts_result.extend(parsed_posts)
+
+                    before_param_number = self.__parse_before_param_value(
+                        post_url=parsed_posts[-1].url
+                    )
+                    if before_param_number <= TelegramConfig.before_param_size:
+                        before_param_number -= TelegramConfig.before_param_size
+                    else:
+                        before_param_number = (
+                            before_param_number - TelegramConfig.before_param_size
+                        )
+                        if before_param_number <= 0:
+                            break
+
+                    preview_url = utils.url.build_param_before_url(
+                        url=preview_url, before=before_param_number
+                    )
+
+                except Exception as e:
+                    await logger.aerror(
+                        f"Error parsing channel {channel_username}: {e}"
+                    )
+                    break
+
+            return posts_result
+
+    async def parse(self):
+        """Main parsing method"""
+        await self.initialize()
+
+        try:
+            for channel_username in self.channels:
+                try:
+                    await self.parse_channel(channel_username)
+                except Exception as e:
+                    await logger.aerror(
+                        f"Failed to parse channel {channel_username}: {e}"
+                    )
+                    continue
+        finally:
+            await self.close()

telegram_pm/parsers/tag_options.py

@@ -0,0 +1,78 @@
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class TagOptions:
+    attrs: dict
+    tag: str
+
+
+class PostParseConfig:
+    channel_id = TagOptions(
+        tag="div",
+        attrs={
+            "class": "tgme_widget_message text_not_supported_wrap js-widget_message"
+        },
+    )
+
+    post_block = TagOptions(
+        tag="div",
+        attrs={"class": re.compile(r"tgme_widget_message_wrap js-widget_message_wrap")},
+    )
+
+    post_url = TagOptions(tag="a", attrs={"class": "tgme_widget_message_date"})
+
+    replied_url = TagOptions(tag="a", attrs={"class": "tgme_widget_message_reply"})
+
+    text = TagOptions(
+        tag="div",
+        attrs={"class": re.compile(r"tgme_widget_message_text js-message_text")},
+    )
+
+    date = TagOptions(tag="time", attrs={"class": "time"})
+
+    photo_url = TagOptions(
+        tag="a", attrs={"class": re.compile(r"tgme_widget_message_photo_wrap")}
+    )
+
+    video_url = TagOptions(
+        tag="i", attrs={"class": re.compile(r"tgme_widget_message_video_thumb")}
+    )
+
+    round_video_url = TagOptions(
+        tag="video",
+        attrs={
+            "class": re.compile(r"tgme_widget_message_roundvideo js-message_roundvideo")
+        },
+    )
+
+    url = TagOptions(
+        tag="a",
+        attrs={
+            "target": re.compile(r"_blank"),
+            "href": re.compile(r"^https?://"),
+        },
+    )
+
+    url_preview = TagOptions(
+        tag="a", attrs={"class": re.compile(r"tgme_widget_message_link_preview")}
+    )
+
+    file = TagOptions(
+        tag="div", attrs={"class": re.compile(r"tgme_widget_message_document_title")}
+    )
+
+    file_extra = TagOptions(
+        tag="div", attrs={"class": re.compile(r"tgme_widget_message_document_extra")}
+    )
+
+    tag = TagOptions(tag="a", attrs={"href": re.compile(r"^\?q=%23")})
+
+    forwarded_from_name = TagOptions(
+        tag="a", attrs={"class": "tgme_widget_message_forwarded_from_name"}
+    )
+
+    forwarded_from_url = TagOptions(
+        tag="a", attrs={"class": "tgme_widget_message_forwarded_from_name"}
+    )

telegram_pm/run.py

@@ -0,0 +1,49 @@
+import sys
+import signal
+import asyncio
+
+from telegram_pm.parsers.preview import PreviewParser
+from telegram_pm.utils.logger import logger
+from telegram_pm.config import TelegramConfig
+
+
+class ParserRunner:
+    def __init__(self, db_path: str, channels: list[str], verbose: bool = False):
+        self.db_path = db_path
+        self.channels = channels
+        self.verbose = verbose
+
+        self._shutdown = False
+
+        # Setup signal handlers
+        signal.signal(signal.SIGINT, self.handle_signal)
+        signal.signal(signal.SIGTERM, self.handle_signal)
+
+    def handle_signal(self, signum, frame):
+        logger.info(f"Received signal {signum}, shutting down...")
+        self._shutdown = True
+        sys.exit(0)
+
+    async def run(self):
+        parser = PreviewParser(
+            channels=self.channels, verbose=self.verbose, db_path=self.db_path
+        )
+        try:
+            while not self._shutdown:
+                try:
+                    await parser.parse()
+                    logger.info(
+                        f"💤 Sleep {TelegramConfig.sleep_time_seconds} seconds ... 💤"
+                    )
+                    await asyncio.sleep(TelegramConfig.sleep_time_seconds)
+                except Exception as e:
+                    logger.error(f"Error during parsing: {e}")
+                    await asyncio.sleep(TelegramConfig.sleep_after_error_request)
+        finally:
+            if parser:
+                await parser.close()
+
+
+def run_parser(db_path: str, channels: list[str], verbose: bool = False):
+    runner = ParserRunner(channels=channels, verbose=verbose, db_path=db_path)
+    asyncio.run(runner.run())

telegram_pm/utils/__init__.py

@@ -0,0 +1,8 @@
+from telegram_pm.utils import logger, url, parse
+
+
+__all__ = [
+    "logger",
+    "url",
+    "parse",
+]

telegram_pm/utils/logger.py

@@ -0,0 +1,5 @@
+import structlog
+from structlog.typing import FilteringBoundLogger
+
+
+logger: FilteringBoundLogger = structlog.get_logger()

telegram_pm/utils/parse.py

@@ -0,0 +1,46 @@
+import re
+import json
+import base64
+
+from bs4 import BeautifulSoup, PageElement
+
+from telegram_pm.parsers.tag_options import TagOptions
+
+
+URL_REGEX = re.compile(
+    r"https?://(?:www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b[-a-zA-Z0-9()@:%_+.~#?&/=]*"
+)
+
+
+def content_to_bs(content: str) -> BeautifulSoup:
+    return BeautifulSoup(content, "html5lib")
+
+
+def extract_element(
+    bs_content: BeautifulSoup, tag_ops: TagOptions
+) -> list[PageElement]:
+    elements = bs_content.find_all(tag_ops.tag, attrs=tag_ops.attrs)
+    return [elem for elem in elements]
+
+
+def extract_url_from_style(style_content: str) -> str | None:
+    url = URL_REGEX.search(style_content)
+    if url:
+        return url.group(0)
+    return None
+
+
+def channel_id_clean(id_str: str) -> int:
+    """
+    Extract id from channel id string
+        c2233445566/14992 -> 2233445566
+    """
+    channel_id = id_str.split("/")[0][1:]
+    return int(channel_id)
+
+
+def decode_channel_id(channel_id_base64: str) -> int:
+    if not channel_id_base64.endswith("="):
+        channel_id_base64 += "=="
+    channel_id = json.loads(base64.b64decode(channel_id_base64))
+    return channel_id["c"]

telegram_pm/utils/url.py

@@ -0,0 +1,34 @@
+from urllib.parse import urljoin
+
+from telegram_pm.config import TelegramConfig
+
+
+def build_preview_url(username: str) -> str:
+    """
+    Build preview URL.
+        username -> https://t.me/s/username
+    :param username: Telegram username
+    :return: str
+    """
+    return urljoin(TelegramConfig.base_url, urljoin("/s/", username))
+
+
+def build_param_before_url(url: str, before: int | str) -> str:
+    """
+    Build preview URL with before parameter.
+        - https://t.me/s/username -> https://t.me/s/username?before
+        - https://t.me/s/username -> https://t.me/s/username?before=123
+    :param url: str - Preview URL
+    :param before: - Before parameter value
+    :return: str
+    """
+    return urljoin(url, f"?before={before}")
+
+
+def get_username_from_tg_url(url: str) -> str:
+    """
+    Get username from Telegram URL.
+    """
+    if url.startswith(TelegramConfig.base_url):
+        return url.split("/")[-1]
+    return url

telegram_pm-0.1.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 aIligat0r
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

telegram_pm-0.1.0.dist-info/METADATA

@@ -0,0 +1,144 @@
+Metadata-Version: 2.3
+Name: telegram-pm
+Version: 0.1.0
+Summary: Telegram preview page parser
+Author: Your Name
+Author-email: you@example.com
+Requires-Python: >=3.12
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: aiosqlite (>=0.21.0,<0.22.0)
+Requires-Dist: bs4 (>=0.0.2,<0.0.3)
+Requires-Dist: html5lib (>=1.1,<2.0)
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: python-dotenv (>=1.1.0,<2.0.0)
+Requires-Dist: retry (>=0.9.2,<0.10.0)
+Requires-Dist: structlog (>=25.2.0,<26.0.0)
+Requires-Dist: typer (>=0.15.2,<0.16.0)
+Description-Content-Type: text/markdown
+
+# Telegram Channels Monitor
+
+![Python](https://img.shields.io/badge/python-3.12%2B-blue)
+![License](https://img.shields.io/badge/license-MIT-green)
+
+Tool for monitoring public Telegram channels available in WEB preview mode
+
+## 🌟 Features
+1. [x] Parsing recent messages from public Telegram channels
+2. [x] Extracting metadata and media attachments
+3. [x] Storing data in SQLite database
+4. [x] Support for forwarded messages and replies
+5. [x] Configurable data collection parameters
+
+## 🛠 Installation
+1. Ensure Python 3.12+ is installed (recommendation)
+2. Clone repository
+```bash
+   git clone 'https://github.com/aIligat0r/tpm.git'
+```
+
+## ⚙️ Configuration
+Configurations (file `.env` or `tpm/config.py`)
+
+Parsing configurations:
+* `TELEGRAM_PARSE_REPEAT_COUNT` - Number of requests (default `5`). 20 messages per request. (1 iter - last 20 messages)
+* `TELEGRAM_SLEEP_TIME_SECONDS` - Number of seconds after which the next process of receiving data from channels will begin (default `60` seconds)
+* `TELEGRAM_SLEEP_AFTER_ERROR_REQUEST` - Waiting after a failed requests (default `30`)
+
+HTTP configurations:
+* `HTTP_RETRIES` - Number of repeated request attempts (default `3`)
+* `HTTP_BACKOFF` - Delay between attempts for failed requests (default `3` seconds)
+* `HTTP_TIMEOUT` - Waiting for a response (default `30` seconds)
+
+## 🚀 Usage
+
+#### 1. Build application:
+
+Build docker image:
+```bash
+docker build -t telegram_pm .
+```
+Create poetry env:
+* Install poetry:
+```bash
+pip install poetry
+```
+* Create poetry env and install packages:
+```bash
+poetry install
+```
+
+#### 2. Launching the app
+
+| Options                           | Description                                                           | Required                                                       |
+|-----------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------|
+| `--db-path`                       | Path to the base (if not, it will be created)                         | ❌ required                                                     |
+| `--channels-filepath`/`--ch-file` | File of channel usernames (file where in each line Telegram username) | ❌ required (or usernames `--channel`/`--ch`)                   |
+| `--channel`/`--ch`                | List of usernames that are passed by the parameter                    | ❌ required (or file of channels `--channels-filepath`/`--chf`) |
+| `--verbose`/`--v`                 | Verbose mode                                                          | ➖                                                              |
+| `--help`/`--h`                    | Help information                                                      | ➖                                                              |
+
+**Poetry**:
+```bash
+poetry run telegram_pm --ch freegaza --ch BREAKINGNewsTG --db-path .\tg.db --v
+```
+or
+```bash
+poetry run telegram_pm --channels-filepath /path/to/monitoring_usernames.txt --db-path .\tg.db
+```
+**Docker**:
+```bash
+docker run -it --rm telegram_pm --ch freegaza --db-path test_tg.db --v
+```
+or (if you want to transfer usernames in a file, then you need to mount the paths)
+```bash
+$ mkdir ~/tpm_data/  # create a folder for data
+$ cp /path/to/channel/usernames.txt ~/tpm_data/usernames.txt  #  copy the file with the user names to the previously created folder
+$ chmod 666 ~/tpm_data_dir/telegram_messages.sqlite && chmod 666 ~/tpm_data_dir/usernames.txt  # grant access to use this folder from the container
+```
+```bash
+docker run -it --rm \
+    -v ~/tpm_data_dir/telegram_messages.sqlite:/data/telegram_messages.sqlite \
+    -v ~/tpm_data_dir/usernames.txt:/data/usernames.txt \
+    telegram_pm --db-path /data/telegram_messages.sqlite --chf /data/usernames.txt
+```
+## 🗃️ Database Structure
+
+The tables will be named as usernames. Each table is a username that was passed in the running parameters.
+
+| Field                 | Type                              | Description                                              |
+|-----------------------|-----------------------------------|----------------------------------------------------------|
+| `id`                  | **INTEGER**                       | Channel ID                                               |
+| `url`                 | **TEXT**                          | Message URL                                              |
+| `username`            | **TEXT**                          | Channel username                                         |
+| `date`                | **TEXT** _(ISO 8601)_             | Message date                                             |
+| `text`                | **TEXT**                          | Message text                                             |
+| `replied_post_url`    | **TEXT**                          | Replied message URL                                      |
+| `urls`                | **JSON**                          | URLs from text                                           |
+| `photo_urls`          | **JSON**                          | Photo URLs                                               |
+| `video_urls`          | **JSON**                          | Video URLs                                               |
+| `created_at`          | **CURRENT_DATETIME** _(ISO 8601)_ | Record creation time                                     |
+| `url_preview`         | **TEXT**                          | Text from preview URL                                    |
+| `round_video_url`     | **TEXT**                          | URL to round video message                               |
+| `files`               | **JSON**                          | List of file names and their description                 |
+| `tags`                | **JSON**                          | List of tags from a message body                         |
+| `forwarded_from_url`  | **TEXT**                          | URL of the channel from which the message was forwarded  |
+| `forwarded_from_name` | **TEXT**                          | Name of the channel from which the message was forwarded |
+
+
+## ⚠️ Limitations
+Works only with public channels
+
+## 🧮 Example of work
+**_Verbose mode:_**
+
+![img.png](img_verbose_sample.png)
+
+**_View database_**
+![img.png](img_view_tables.png)
+
+## 📜 License
+MIT License
+

telegram_pm-0.1.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+telegram_pm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+telegram_pm/config.py,sha256=Qa7QMgRpYiGPZ8dmsSR6eYSxXuKoSGmiGG1mMe4Rmmc,765
+telegram_pm/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+telegram_pm/database/db.py,sha256=rSfqCbBYrD4E_Msb5q8ilY1QIPlq7vnVE_-dNlYOXaM,4716
+telegram_pm/entities.py,sha256=-mdx3u1M7bKFtEXaLcaaBjLQg08NBW77c2VeNHQQ_Gw,646
+telegram_pm/http_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+telegram_pm/http_client/client.py,sha256=xvBsoFDo4QaQtgcmpNr3NjtKnyryL7HSn71DR89Quec,718
+telegram_pm/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+telegram_pm/parsers/base.py,sha256=9GH7bJaqPueohRoK1OVOmjF9pY_fqaFizIc9Ak6qS-Y,22
+telegram_pm/parsers/post.py,sha256=4wf4KBG6NBOFGpk8_GH88M1hbyjTWXTQgRgGaHXgB40,10469
+telegram_pm/parsers/preview.py,sha256=Z9xdbFPktQQzTyfEzw0M0vANHXYZkfmarZAoXDfDl5I,6055
+telegram_pm/parsers/tag_options.py,sha256=0YRQH5O8fpfReHRDXEThmFFyiacsUz-wlbjVFOLoiJ8,2040
+telegram_pm/run.py,sha256=8g2AWgAEr6uqcYuEh67WyA3BXv_WuccwBUTaQ8blJU4,1653
+telegram_pm/utils/__init__.py,sha256=loG7JOo8Th7vV7lYrVeCEhObguEaMQr7xRCmVkV7CM4,103
+telegram_pm/utils/logger.py,sha256=RqwcrFNMzjQfqB-aC9w79g9WLbcj6GvokRDtj9ZPH1Y,123
+telegram_pm/utils/parse.py,sha256=vSI4kNVvt2hqXLcOdp0MuCChG6fFqSrb17VzH6huqVQ,1167
+telegram_pm/utils/url.py,sha256=mv5Lc4PZbyL4hQXku3sGzMt3lmGKjtlYhbmzL0fKeb8,941
+telegram_pm-0.1.0.dist-info/entry_points.txt,sha256=dIvBN0V4aMrJKl7tB1qCYy7VM40uFqnuwcPibXfnSU0,40
+telegram_pm-0.1.0.dist-info/LICENSE,sha256=kaLyGzbJPljgIIJrGiWc2611z1YfjYG8QsI6v0C_oug,1066
+telegram_pm-0.1.0.dist-info/METADATA,sha256=ea4tZgKbhsV1dpvJWLmamzEOEKYDaagzU4xCyQt1Go8,7053
+telegram_pm-0.1.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+telegram_pm-0.1.0.dist-info/RECORD,,

telegram_pm-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.2
+Root-Is-Purelib: true
+Tag: py3-none-any

telegram_pm-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+tpm=commands.cli:app
+