telegram-pm 0.1.3__tar.gz → 0.1.5__tar.gz

{telegram_pm-0.1.3 → telegram_pm-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: telegram-pm
-Version: 0.1.3
+Version: 0.1.5
 Summary: Telegram preview page parser
 Author: Your Name
 Author-email: you@example.com
@@ -37,7 +37,7 @@ Telegram monitoring tool for public channels that can be viewed via WEB preview.
 1. Ensure Python 3.12+ is installed (recommendation)
 2. Clone repository
 ```bash
-   git clone 'https://github.com/aIligat0r/tpm.git'
+git clone 'https://github.com/aIligat0r/tpm.git'
 ```
 or
 ```bash
@@ -77,13 +77,14 @@ poetry install
 
 #### 2. Launching the app
 
-| Options                           | Description                                                           | Required                                                       |
-|-----------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------|
-| `--db-path`                       | Path to the base (if not, it will be created)                         | ❌ required                                                     |
-| `--channels-filepath`/`--ch-file` | File of channel usernames (file where in each line Telegram username) | ❌ required (or usernames `--channel`/`--ch`)                   |
-| `--channel`/`--ch`                | List of usernames that are passed by the parameter                    | ❌ required (or file of channels `--channels-filepath`/`--chf`) |
-| `--verbose`/`--v`                 | Verbose mode                                                          | ➖                                                              |
-| `--help`/`--h`                    | Help information                                                      | ➖                                                              |
+| Options                       | Description                                                           | Required                                                       |
+|-------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------|
+| `--db-path`                   | Path to db file (if sqlite). Else path to dir (if csv)                | ❌ required                                                     |
+| `--channels-filepath`/`--chf` | File of channel usernames (file where in each line Telegram username) | ❌ required (or usernames `--channel`/`--ch`)                   |
+| `--channel`/`--ch`            | List of usernames that are passed by the parameter                    | ❌ required (or file of channels `--channels-filepath`/`--chf`) |
+| `--verbose`/`--v`             | Verbose mode                                                          | ➖                                                              |
+| `--format`/`--f`              | Data saving format (csv, sqlite)                                      | ➖                                                              |
+| `--help`/`--h`                | Help information                                                      | ➖                                                              |
 
 **Poetry:**
 ```bash
@@ -107,7 +108,7 @@ $ chmod 666 ~/tpm_data_dir/telegram_messages.sqlite && chmod 666 ~/tpm_data_dir/
 docker run -it --rm \
     -v ~/tpm_data_dir/telegram_messages.sqlite:/data/telegram_messages.sqlite \
     -v ~/tpm_data_dir/usernames.txt:/data/usernames.txt \
-    telegram_pm --db-path /data/telegram_messages.sqlite --chf /data/usernames.txt
+    tpm --db-path /data/telegram_messages.sqlite --chf /data/usernames.txt
 ```
 **Python:**
 ```python
@@ -115,11 +116,12 @@ from telegram_pm.run import run_tpm
 
 
 run_tpm(
-    db_path="tg.db",                    # Path to sqlite database
+    db_path="tg.db",                    # Path to db file (if sqlite). Else path to dir (if csv)
     channels=["channel1", "channel2"],  # Channels list
     verbose=True,                       # Verbose mode
 
     # Configuration (optional)
+    format="sqlite",                    # Data saving format (csv, sqlite)
     tg_iteration_in_preview_count=5,    # Number of requests (default 5). 20 messages per request. (1 iter - last 20 messages)
     tg_sleep_time_seconds=60,           # Number of seconds after which the next process of receiving data from channels will begin (default 60 seconds)
     tg_sleep_after_error_request=30,    # Waiting after a failed requests (default 30)

{telegram_pm-0.1.3 → telegram_pm-0.1.5}/README.md

@@ -17,7 +17,7 @@ Telegram monitoring tool for public channels that can be viewed via WEB preview.
 1. Ensure Python 3.12+ is installed (recommendation)
 2. Clone repository
 ```bash
-   git clone 'https://github.com/aIligat0r/tpm.git'
+git clone 'https://github.com/aIligat0r/tpm.git'
 ```
 or
 ```bash
@@ -57,13 +57,14 @@ poetry install
 
 #### 2. Launching the app
 
-| Options                           | Description                                                           | Required                                                       |
-|-----------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------|
-| `--db-path`                       | Path to the base (if not, it will be created)                         | ❌ required                                                     |
-| `--channels-filepath`/`--ch-file` | File of channel usernames (file where in each line Telegram username) | ❌ required (or usernames `--channel`/`--ch`)                   |
-| `--channel`/`--ch`                | List of usernames that are passed by the parameter                    | ❌ required (or file of channels `--channels-filepath`/`--chf`) |
-| `--verbose`/`--v`                 | Verbose mode                                                          | ➖                                                              |
-| `--help`/`--h`                    | Help information                                                      | ➖                                                              |
+| Options                       | Description                                                           | Required                                                       |
+|-------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------|
+| `--db-path`                   | Path to db file (if sqlite). Else path to dir (if csv)                | ❌ required                                                     |
+| `--channels-filepath`/`--chf` | File of channel usernames (file where in each line Telegram username) | ❌ required (or usernames `--channel`/`--ch`)                   |
+| `--channel`/`--ch`            | List of usernames that are passed by the parameter                    | ❌ required (or file of channels `--channels-filepath`/`--chf`) |
+| `--verbose`/`--v`             | Verbose mode                                                          | ➖                                                              |
+| `--format`/`--f`              | Data saving format (csv, sqlite)                                      | ➖                                                              |
+| `--help`/`--h`                | Help information                                                      | ➖                                                              |
 
 **Poetry:**
 ```bash
@@ -87,7 +88,7 @@ $ chmod 666 ~/tpm_data_dir/telegram_messages.sqlite && chmod 666 ~/tpm_data_dir/
 docker run -it --rm \
     -v ~/tpm_data_dir/telegram_messages.sqlite:/data/telegram_messages.sqlite \
     -v ~/tpm_data_dir/usernames.txt:/data/usernames.txt \
-    telegram_pm --db-path /data/telegram_messages.sqlite --chf /data/usernames.txt
+    tpm --db-path /data/telegram_messages.sqlite --chf /data/usernames.txt
 ```
 **Python:**
 ```python
@@ -95,11 +96,12 @@ from telegram_pm.run import run_tpm
 
 
 run_tpm(
-    db_path="tg.db",                    # Path to sqlite database
+    db_path="tg.db",                    # Path to db file (if sqlite). Else path to dir (if csv)
     channels=["channel1", "channel2"],  # Channels list
     verbose=True,                       # Verbose mode
 
     # Configuration (optional)
+    format="sqlite",                    # Data saving format (csv, sqlite)
     tg_iteration_in_preview_count=5,    # Number of requests (default 5). 20 messages per request. (1 iter - last 20 messages)
     tg_sleep_time_seconds=60,           # Number of seconds after which the next process of receiving data from channels will begin (default 60 seconds)
     tg_sleep_after_error_request=30,    # Waiting after a failed requests (default 30)

{telegram_pm-0.1.3 → telegram_pm-0.1.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "telegram-pm"
-version = "0.1.3"
+version = "0.1.5"
 description = "Telegram preview page parser"
 authors = [{name = "Your Name",email = "you@example.com"}]
 readme = "README.md"
@@ -8,7 +8,7 @@ requires-python = ">=3.12"
 packages = [{ include = "src" }, { include = "commands" }]
 
 [tool.poetry.scripts]
-tpm = "commands.cli:app"
+telegram-pm = "commands.cli:app"
 
 [tool.poetry.dependencies]
 httpx = "^0.28.1"
@@ -24,6 +24,7 @@ aiosqlite = "^0.21.0"
 ruff = "^0.11.5"
 pre-commit = "^4.2.0"
 mypy = "^1.15.0"
+pytest = "^8.3.5"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

telegram_pm-0.1.5/telegram_pm/database/csv_processor.py

@@ -0,0 +1,94 @@
+import csv
+from typing import List
+from dataclasses import asdict
+from pathlib import Path
+import json
+from datetime import datetime
+
+from telegram_pm.entities import Post
+
+
+class CSVProcessor:
+    def __init__(self, csv_dir: str):
+        self.csv_dir = Path(csv_dir)
+        self.csv_dir.mkdir(parents=True, exist_ok=True)
+
+    def _get_filename(self, table_name: str) -> Path:
+        return self.csv_dir / f"{table_name}.csv"
+
+    async def table_exists(self, table_name: str) -> bool:
+        return self._get_filename(table_name).exists()
+
+    async def create_table_from_post(self, table_name: str):
+        filename = self._get_filename(table_name)
+        if not filename.exists():
+            filename.touch()
+
+    async def insert_posts_batch(self, table_name: str, posts: List[Post]):
+        if not posts:
+            return
+
+        filename = self._get_filename(table_name)
+        file_exists = filename.exists()
+
+        columns = [
+            "url",
+            "username",
+            "id",
+            "date",
+            "text",
+            "replied_post_url",
+            "urls",
+            "url_preview",
+            "photo_urls",
+            "video_urls",
+            "round_video_url",
+            "files",
+            "tags",
+            "created_at",
+            "forwarded_from_url",
+            "forwarded_from_name",
+        ]
+
+        with open(filename, "a", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=columns)
+
+            if not file_exists or filename.stat().st_size == 0:
+                writer.writeheader()
+
+            for post in posts:
+                post_dict = asdict(post)
+                for field in ["urls", "photo_urls", "video_urls", "files", "tags"]:
+                    post_dict[field] = json.dumps(post_dict[field])
+                if "created_at" not in post_dict or not post_dict["created_at"]:
+                    post_dict["created_at"] = datetime.now().isoformat()
+                writer.writerow(post_dict)
+
+    async def is_table_empty(self, table_name: str) -> bool:
+        filename = self._get_filename(table_name)
+        if not filename.exists():
+            return True
+        return filename.stat().st_size == 0
+
+    async def drop_table_if_empty(self, table_name: str):
+        filename = self._get_filename(table_name)
+        if await self.is_table_empty(table_name) and filename.exists():
+            filename.unlink()
+
+    async def post_exists(self, table_name: str, url: str) -> bool:
+        filename = self._get_filename(table_name)
+        if not filename.exists():
+            return False
+
+        with open(filename, "r", newline="", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                if row["url"] == url:
+                    return True
+        return False
+
+    async def close(self):
+        pass
+
+    async def initialize(self):
+        pass

{telegram_pm-0.1.3 → telegram_pm-0.1.5}/telegram_pm/parsers/preview.py

@@ -10,7 +10,8 @@ from telegram_pm.utils.logger import logger
 from telegram_pm.parsers.base import BaseParser
 from telegram_pm.parsers.post import PostsParser
 from telegram_pm.http_client.client import HttpClient
-from telegram_pm.database.db import DatabaseProcessor
+from telegram_pm.database.sqlite_processor import DatabaseProcessor
+from telegram_pm.database.csv_processor import CSVProcessor
 
 
 class PreviewParser(BaseParser):
@@ -22,6 +23,7 @@ class PreviewParser(BaseParser):
         self,
         channels: list[str],
         db_path: str,
+        format: str = "sqlite",
         verbose: bool = False,
         tg_before_param_size: int = config.TelegramConfig.before_param_size,
         tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
@@ -58,10 +60,17 @@ class PreviewParser(BaseParser):
             headers=http_headers,
         )
         self.post_parser = PostsParser(verbose=verbose)
-        self.db = DatabaseProcessor(db_path=db_path)
+        self.db = self.__initial_db(format=format, db_path=db_path)
         self._db_initialized = False
         self.verbose = verbose
 
+    @staticmethod
+    def __initial_db(format: str, db_path: str):
+        if format == "sqlite":
+            return DatabaseProcessor(db_path=db_path)
+        else:
+            return CSVProcessor(csv_dir=db_path)
+
     @staticmethod
     def __forbidden_parse_preview(response: httpx.Response) -> bool:
         """

{telegram_pm-0.1.3 → telegram_pm-0.1.5}/telegram_pm/run.py

@@ -27,6 +27,7 @@ class ParserRunner:
         self,
         db_path: str,
         channels: list[str],
+        format: str = "sqlite",
         verbose: bool = False,
         tg_before_param_size: int = config.TelegramConfig.before_param_size,
         tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
@@ -51,6 +52,7 @@ class ParserRunner:
         :param http_headers: HTTP headers
         """
         parser = PreviewParser(
+            format=format,
             channels=channels,
             verbose=verbose,
             db_path=db_path,
@@ -82,6 +84,7 @@ class ParserRunner:
 def run_tpm(
     db_path: str,
     channels: list[str],
+    format: str,
     verbose: bool = False,
     tg_before_param_size: int = config.TelegramConfig.before_param_size,
     tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
@@ -95,6 +98,7 @@ def run_tpm(
     runner = ParserRunner()
     asyncio.run(
         runner.run(
+            format=format,
             channels=channels,
             verbose=verbose,
             db_path=db_path,