telegram-pm 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

telegram_pm/config.py

@@ -12,6 +12,9 @@ class HttpClientConfig(BaseConfig):
     retries: int = int(environ.get("HTTP_RETRIES", 3))
     backoff: int = int(environ.get("HTTP_BACKOFF", 3))
     timeout: int = int(environ.get("HTTP_TIMEOUT", 30))
+    headers: dict[str, str] = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+    }
 
 
 class TelegramConfig(BaseConfig):

telegram_pm/http_client/client.py

@@ -6,17 +6,31 @@ from telegram_pm.config import HttpClientConfig
 
 
 class HttpClient:
-    def __init__(self):
+    def __init__(
+        self,
+        retries: int = HttpClientConfig.retries,
+        timeout: int = HttpClientConfig.timeout,
+        backoff: int = HttpClientConfig.backoff,
+        headers: dict[str, str] = HttpClientConfig.headers,
+    ):
+        self._headers = headers
+        self._backoff = backoff
+        self._retries = retries
         self.client = httpx.AsyncClient(
             transport=httpx.AsyncHTTPTransport(
                 verify=False,
-                retries=HttpClientConfig.retries,
+                retries=retries,
             ),
-            timeout=HttpClientConfig.timeout,
+            timeout=timeout,
             verify=False,
         )
 
-    @retry(backoff=HttpClientConfig.backoff, logger=logger)  # type: ignore[arg-type]
     async def request(self, url: str, method: str = "GET", **kwargs) -> httpx.Response:
-        response = await self.client.request(method=method, url=url, **kwargs)
-        return response
+        @retry(backoff=self._backoff, logger=logger)  # type: ignore[arg-type]
+        async def nested_request() -> httpx.Response:
+            response = await self.client.request(
+                method=method, url=url, headers=self._headers, **kwargs
+            )
+            return response
+
+        return await nested_request()

telegram_pm/parsers/preview.py

@@ -4,10 +4,9 @@ import httpx
 from bs4 import BeautifulSoup
 from structlog.contextvars import bound_contextvars
 
-from telegram_pm import utils
+from telegram_pm import utils, config
 from telegram_pm.entities import Post
 from telegram_pm.utils.logger import logger
-from telegram_pm.config import TelegramConfig
 from telegram_pm.parsers.base import BaseParser
 from telegram_pm.parsers.post import PostsParser
 from telegram_pm.http_client.client import HttpClient
@@ -24,9 +23,40 @@ class PreviewParser(BaseParser):
         channels: list[str],
         db_path: str,
         verbose: bool = False,
+        tg_before_param_size: int = config.TelegramConfig.before_param_size,
+        tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
+        tg_sleep_time_seconds: int = config.TelegramConfig.sleep_time_seconds,
+        tg_sleep_after_error_request: int = config.TelegramConfig.sleep_after_error_request,
+        http_retries: int = config.HttpClientConfig.retries,
+        http_backoff: int = config.HttpClientConfig.backoff,
+        http_timeout: int = config.HttpClientConfig.timeout,
+        http_headers: dict[str, str] = config.HttpClientConfig.headers,
     ):
+        """
+        :param db_path: Path to sqlite database
+        :param channels: Channels list
+        :param verbose: Verbose mode
+        :param tg_before_param_size: 20 messages per request. (1 iter - last 20 messages)
+        :param tg_iteration_in_preview_count: Number of requests (default 5). 20 messages per request. (1 iter - last 20 messages)
+        :param tg_sleep_time_seconds:  Number of seconds after which the next process of receiving data from channels will begin (default 60 seconds)
+        :param tg_sleep_after_error_request: Waiting after a failed requests (default 30)
+        :param http_retries: Number of repeated request attempts (default 3)
+        :param http_backoff: Delay between attempts for failed requests (default 3 seconds)
+        :param http_timeout: Waiting for a response (default 30 seconds)
+        :param http_headers: HTTP headers
+        """
+        self._tg_sleep_after_error_request = tg_sleep_after_error_request
+        self._tg_sleep_time_seconds = tg_sleep_time_seconds
+        self._tg_iteration_in_preview_count = tg_iteration_in_preview_count
+        self._tg_before_param_size = tg_before_param_size
+
         self.channels: list[str] = channels
-        self.http_client = HttpClient()
+        self.http_client = HttpClient(
+            retries=http_retries,
+            backoff=http_backoff,
+            timeout=http_timeout,
+            headers=http_headers,
+        )
         self.post_parser = PostsParser(verbose=verbose)
         self.db = DatabaseProcessor(db_path=db_path)
         self._db_initialized = False
@@ -89,7 +119,7 @@ class PreviewParser(BaseParser):
             posts_result = []
             should_break = False
 
-            for parse_repeat in range(TelegramConfig.iteration_in_preview_count):
+            for parse_repeat in range(self._tg_iteration_in_preview_count):
                 if should_break:
                     await logger.ainfo("No new posts yet")
                     break
@@ -98,7 +128,7 @@ class PreviewParser(BaseParser):
                     response = await self._get_preview_page(preview_url=preview_url)
                     if not response:
                         await logger.awarning("Can not get preview page")
-                        await asyncio.sleep(TelegramConfig.sleep_after_error_request)
+                        await asyncio.sleep(self._tg_sleep_after_error_request)
                         continue
 
                     if self.__forbidden_parse_preview(response=response):
@@ -111,7 +141,7 @@ class PreviewParser(BaseParser):
                     if not parsed_posts:
                         await logger.awarning("No posts parsed from preview page")  # type: ignore
                         await self.db.drop_table_if_empty(channel_username)
-                        await asyncio.sleep(TelegramConfig.sleep_after_error_request)
+                        await asyncio.sleep(self._tg_sleep_after_error_request)
                         break
 
                     first_post_exists = await self.db.post_exists(
@@ -127,11 +157,11 @@ class PreviewParser(BaseParser):
                     before_param_number = self.__parse_before_param_value(
                         post_url=parsed_posts[-1].url
                     )
-                    if before_param_number <= TelegramConfig.before_param_size:
-                        before_param_number -= TelegramConfig.before_param_size
+                    if before_param_number <= self._tg_before_param_size:
+                        before_param_number -= self._tg_before_param_size
                     else:
                         before_param_number = (
-                            before_param_number - TelegramConfig.before_param_size
+                            before_param_number - self._tg_before_param_size
                         )
                         if before_param_number <= 0:
                             break

telegram_pm/run.py

@@ -2,17 +2,16 @@ import sys
 import signal
 import asyncio
 
-from telegram_pm.parsers.preview import PreviewParser
+from telegram_pm import config
 from telegram_pm.utils.logger import logger
 from telegram_pm.config import TelegramConfig
+from telegram_pm.parsers.preview import PreviewParser
 
 
 class ParserRunner:
-    def __init__(self, db_path: str, channels: list[str], verbose: bool = False):
-        self.db_path = db_path
-        self.channels = channels
-        self.verbose = verbose
-
+    def __init__(
+        self,
+    ):
         self._shutdown = False
 
         # Setup signal handlers
@@ -24,9 +23,45 @@ class ParserRunner:
         self._shutdown = True
         sys.exit(0)
 
-    async def run(self):
+    async def run(
+        self,
+        db_path: str,
+        channels: list[str],
+        verbose: bool = False,
+        tg_before_param_size: int = config.TelegramConfig.before_param_size,
+        tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
+        tg_sleep_time_seconds: int = config.TelegramConfig.sleep_time_seconds,
+        tg_sleep_after_error_request: int = config.TelegramConfig.sleep_after_error_request,
+        http_retries: int = config.HttpClientConfig.retries,
+        http_backoff: int = config.HttpClientConfig.backoff,
+        http_timeout: int = config.HttpClientConfig.timeout,
+        http_headers: dict[str, str] = config.HttpClientConfig.headers,
+    ):
+        """
+        :param db_path: Path to sqlite database
+        :param channels: Channels list
+        :param verbose: Verbose mode
+        :param tg_before_param_size: 20 messages per request. (1 iter - last 20 messages)
+        :param tg_iteration_in_preview_count: Number of requests (default 5). 20 messages per request. (1 iter - last 20 messages)
+        :param tg_sleep_time_seconds:  Number of seconds after which the next process of receiving data from channels will begin (default 60 seconds)
+        :param tg_sleep_after_error_request: Waiting after a failed requests (default 30)
+        :param http_retries: Number of repeated request attempts (default 3)
+        :param http_backoff: Delay between attempts for failed requests (default 3 seconds)
+        :param http_timeout: Waiting for a response (default 30 seconds)
+        :param http_headers: HTTP headers
+        """
         parser = PreviewParser(
-            channels=self.channels, verbose=self.verbose, db_path=self.db_path
+            channels=channels,
+            verbose=verbose,
+            db_path=db_path,
+            tg_before_param_size=tg_before_param_size,
+            tg_iteration_in_preview_count=tg_iteration_in_preview_count,
+            tg_sleep_time_seconds=tg_sleep_time_seconds,
+            tg_sleep_after_error_request=tg_sleep_after_error_request,
+            http_retries=http_retries,
+            http_backoff=http_backoff,
+            http_timeout=http_timeout,
+            http_headers=http_headers,
         )
         try:
             while not self._shutdown:
@@ -44,6 +79,32 @@ class ParserRunner:
                 await parser.close()
 
 
-def run_parser(db_path: str, channels: list[str], verbose: bool = False):
-    runner = ParserRunner(channels=channels, verbose=verbose, db_path=db_path)
-    asyncio.run(runner.run())
+def run_tpm(
+    db_path: str,
+    channels: list[str],
+    verbose: bool = False,
+    tg_before_param_size: int = config.TelegramConfig.before_param_size,
+    tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
+    tg_sleep_time_seconds: int = config.TelegramConfig.sleep_time_seconds,
+    tg_sleep_after_error_request: int = config.TelegramConfig.sleep_after_error_request,
+    http_retries: int = config.HttpClientConfig.retries,
+    http_backoff: int = config.HttpClientConfig.backoff,
+    http_timeout: int = config.HttpClientConfig.timeout,
+    http_headers: dict[str, str] = config.HttpClientConfig.headers,
+):
+    runner = ParserRunner()
+    asyncio.run(
+        runner.run(
+            channels=channels,
+            verbose=verbose,
+            db_path=db_path,
+            tg_before_param_size=tg_before_param_size,
+            tg_iteration_in_preview_count=tg_iteration_in_preview_count,
+            tg_sleep_time_seconds=tg_sleep_time_seconds,
+            tg_sleep_after_error_request=tg_sleep_after_error_request,
+            http_retries=http_retries,
+            http_backoff=http_backoff,
+            http_timeout=http_timeout,
+            http_headers=http_headers,
+        )
+    )

{telegram_pm-0.1.0.dist-info → telegram_pm-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: telegram-pm
-Version: 0.1.0
+Version: 0.1.2
 Summary: Telegram preview page parser
 Author: Your Name
 Author-email: you@example.com
@@ -32,15 +32,20 @@ Tool for monitoring public Telegram channels available in WEB preview mode
 4. [x] Support for forwarded messages and replies
 5. [x] Configurable data collection parameters
 
+
 ## 🛠 Installation
 1. Ensure Python 3.12+ is installed (recommendation)
 2. Clone repository
 ```bash
    git clone 'https://github.com/aIligat0r/tpm.git'
 ```
+or
+```bash
+pip install telegram-pm
+```
 
 ## ⚙️ Configuration
-Configurations (file `.env` or `tpm/config.py`)
+Configurations (file `.env` or `telegram_pm/config.py`)
 
 Parsing configurations:
 * `TELEGRAM_PARSE_REPEAT_COUNT` - Number of requests (default `5`). 20 messages per request. (1 iter - last 20 messages)
@@ -58,7 +63,7 @@ HTTP configurations:
 
 Build docker image:
 ```bash
-docker build -t telegram_pm .
+docker build -t tpm .
 ```
 Create poetry env:
 * Install poetry:
@@ -80,17 +85,17 @@ poetry install
 | `--verbose`/`--v`                 | Verbose mode                                                          | ➖                                                              |
 | `--help`/`--h`                    | Help information                                                      | ➖                                                              |
 
-**Poetry**:
+**Poetry:**
 ```bash
-poetry run telegram_pm --ch freegaza --ch BREAKINGNewsTG --db-path .\tg.db --v
+poetry run tpm --ch freegaza --ch BREAKINGNewsTG --db-path .\tg.db --v
 ```
 or
 ```bash
-poetry run telegram_pm --channels-filepath /path/to/monitoring_usernames.txt --db-path .\tg.db
+poetry run tpm --channels-filepath /path/to/monitoring_usernames.txt --db-path .\tg.db
 ```
-**Docker**:
+**Docker:**
 ```bash
-docker run -it --rm telegram_pm --ch freegaza --db-path test_tg.db --v
+docker run -it --rm tpm --ch freegaza --db-path test_tg.db --v
 ```
 or (if you want to transfer usernames in a file, then you need to mount the paths)
 ```bash
@@ -104,6 +109,29 @@ docker run -it --rm \
     -v ~/tpm_data_dir/usernames.txt:/data/usernames.txt \
     telegram_pm --db-path /data/telegram_messages.sqlite --chf /data/usernames.txt
 ```
+**Python:**
+```python
+from telegram_pm.run import run_tpm
+
+
+run_tpm(
+    db_path="tg.db",                    # Path to sqlite database
+    channels=["channel1", "channel2"],  # Channels list
+    verbose=True,                       # Verbose mode
+
+    # Configuration (optional)
+    tg_iteration_in_preview_count=5,    # Number of requests (default 5). 20 messages per request. (1 iter - last 20 messages)
+    tg_sleep_time_seconds=60,           # Number of seconds after which the next process of receiving data from channels will begin (default 60 seconds)
+    tg_sleep_after_error_request=30,    # Waiting after a failed requests (default 30)
+    http_retries=3,                     # Number of repeated request attempts (default 3)
+    http_backoff=3,                     # Delay between attempts for failed requests (default 3 seconds)
+    http_timeout=60,                    # Waiting for a response (default 30 seconds)
+    http_headers={                      # HTTP headers
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+    }
+)
+```
+
 ## 🗃️ Database Structure
 
 The tables will be named as usernames. Each table is a username that was passed in the running parameters.

{telegram_pm-0.1.0.dist-info → telegram_pm-0.1.2.dist-info}/RECORD

@@ -1,22 +1,22 @@
 telegram_pm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-telegram_pm/config.py,sha256=Qa7QMgRpYiGPZ8dmsSR6eYSxXuKoSGmiGG1mMe4Rmmc,765
+telegram_pm/config.py,sha256=w1BZPxy8adyUnVQeGjUseSlVNRgpf7ZGXi4ltCXIo1Y,939
 telegram_pm/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 telegram_pm/database/db.py,sha256=rSfqCbBYrD4E_Msb5q8ilY1QIPlq7vnVE_-dNlYOXaM,4716
 telegram_pm/entities.py,sha256=-mdx3u1M7bKFtEXaLcaaBjLQg08NBW77c2VeNHQQ_Gw,646
 telegram_pm/http_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-telegram_pm/http_client/client.py,sha256=xvBsoFDo4QaQtgcmpNr3NjtKnyryL7HSn71DR89Quec,718
+telegram_pm/http_client/client.py,sha256=EYFiCFZcICntF7Lc1QHsqQ_CcGtNI6G8j-DLmt1VJG4,1149
 telegram_pm/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 telegram_pm/parsers/base.py,sha256=9GH7bJaqPueohRoK1OVOmjF9pY_fqaFizIc9Ak6qS-Y,22
 telegram_pm/parsers/post.py,sha256=4wf4KBG6NBOFGpk8_GH88M1hbyjTWXTQgRgGaHXgB40,10469
-telegram_pm/parsers/preview.py,sha256=Z9xdbFPktQQzTyfEzw0M0vANHXYZkfmarZAoXDfDl5I,6055
+telegram_pm/parsers/preview.py,sha256=TvWy36NOWvMMms3vUdc96wGRuYCvHI8R896gxiKrnJQ,7887
 telegram_pm/parsers/tag_options.py,sha256=0YRQH5O8fpfReHRDXEThmFFyiacsUz-wlbjVFOLoiJ8,2040
-telegram_pm/run.py,sha256=8g2AWgAEr6uqcYuEh67WyA3BXv_WuccwBUTaQ8blJU4,1653
+telegram_pm/run.py,sha256=dKMw2IrtCh3rkkyiwQHNQwRM97f887Z_LsKvUulomrI,4608
 telegram_pm/utils/__init__.py,sha256=loG7JOo8Th7vV7lYrVeCEhObguEaMQr7xRCmVkV7CM4,103
 telegram_pm/utils/logger.py,sha256=RqwcrFNMzjQfqB-aC9w79g9WLbcj6GvokRDtj9ZPH1Y,123
 telegram_pm/utils/parse.py,sha256=vSI4kNVvt2hqXLcOdp0MuCChG6fFqSrb17VzH6huqVQ,1167
 telegram_pm/utils/url.py,sha256=mv5Lc4PZbyL4hQXku3sGzMt3lmGKjtlYhbmzL0fKeb8,941
-telegram_pm-0.1.0.dist-info/entry_points.txt,sha256=dIvBN0V4aMrJKl7tB1qCYy7VM40uFqnuwcPibXfnSU0,40
-telegram_pm-0.1.0.dist-info/LICENSE,sha256=kaLyGzbJPljgIIJrGiWc2611z1YfjYG8QsI6v0C_oug,1066
-telegram_pm-0.1.0.dist-info/METADATA,sha256=ea4tZgKbhsV1dpvJWLmamzEOEKYDaagzU4xCyQt1Go8,7053
-telegram_pm-0.1.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-telegram_pm-0.1.0.dist-info/RECORD,,
+telegram_pm-0.1.2.dist-info/entry_points.txt,sha256=dIvBN0V4aMrJKl7tB1qCYy7VM40uFqnuwcPibXfnSU0,40
+telegram_pm-0.1.2.dist-info/LICENSE,sha256=kaLyGzbJPljgIIJrGiWc2611z1YfjYG8QsI6v0C_oug,1066
+telegram_pm-0.1.2.dist-info/METADATA,sha256=VY6WAWEI_pn83xycXPq-1d8k1kMx8tHKdulFIi2VHSA,8199
+telegram_pm-0.1.2.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+telegram_pm-0.1.2.dist-info/RECORD,,