telegram-pm 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

telegram_pm/database/csv_processor.py

@@ -0,0 +1,94 @@
+import csv
+from typing import List
+from dataclasses import asdict
+from pathlib import Path
+import json
+from datetime import datetime
+
+from telegram_pm.entities import Post
+
+
+class CSVProcessor:
+    def __init__(self, csv_dir: str):
+        self.csv_dir = Path(csv_dir)
+        self.csv_dir.mkdir(parents=True, exist_ok=True)
+
+    def _get_filename(self, table_name: str) -> Path:
+        return self.csv_dir / f"{table_name}.csv"
+
+    async def table_exists(self, table_name: str) -> bool:
+        return self._get_filename(table_name).exists()
+
+    async def create_table_from_post(self, table_name: str):
+        filename = self._get_filename(table_name)
+        if not filename.exists():
+            filename.touch()
+
+    async def insert_posts_batch(self, table_name: str, posts: List[Post]):
+        if not posts:
+            return
+
+        filename = self._get_filename(table_name)
+        file_exists = filename.exists()
+
+        columns = [
+            "url",
+            "username",
+            "id",
+            "date",
+            "text",
+            "replied_post_url",
+            "urls",
+            "url_preview",
+            "photo_urls",
+            "video_urls",
+            "round_video_url",
+            "files",
+            "tags",
+            "created_at",
+            "forwarded_from_url",
+            "forwarded_from_name",
+        ]
+
+        with open(filename, "a", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=columns)
+
+            if not file_exists or filename.stat().st_size == 0:
+                writer.writeheader()
+
+            for post in posts:
+                post_dict = asdict(post)
+                for field in ["urls", "photo_urls", "video_urls", "files", "tags"]:
+                    post_dict[field] = json.dumps(post_dict[field])
+                if "created_at" not in post_dict or not post_dict["created_at"]:
+                    post_dict["created_at"] = datetime.now().isoformat()
+                writer.writerow(post_dict)
+
+    async def is_table_empty(self, table_name: str) -> bool:
+        filename = self._get_filename(table_name)
+        if not filename.exists():
+            return True
+        return filename.stat().st_size == 0
+
+    async def drop_table_if_empty(self, table_name: str):
+        filename = self._get_filename(table_name)
+        if await self.is_table_empty(table_name) and filename.exists():
+            filename.unlink()
+
+    async def post_exists(self, table_name: str, url: str) -> bool:
+        filename = self._get_filename(table_name)
+        if not filename.exists():
+            return False
+
+        with open(filename, "r", newline="", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                if row["url"] == url:
+                    return True
+        return False
+
+    async def close(self):
+        pass
+
+    async def initialize(self):
+        pass

telegram_pm/parsers/preview.py

@@ -10,7 +10,8 @@ from telegram_pm.utils.logger import logger
 from telegram_pm.parsers.base import BaseParser
 from telegram_pm.parsers.post import PostsParser
 from telegram_pm.http_client.client import HttpClient
-from telegram_pm.database.db import DatabaseProcessor
+from telegram_pm.database.sqlite_processor import DatabaseProcessor
+from telegram_pm.database.csv_processor import CSVProcessor
 
 
 class PreviewParser(BaseParser):
@@ -22,6 +23,7 @@ class PreviewParser(BaseParser):
         self,
         channels: list[str],
         db_path: str,
+        format: str = "sqlite",
         verbose: bool = False,
         tg_before_param_size: int = config.TelegramConfig.before_param_size,
         tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
@@ -58,10 +60,17 @@ class PreviewParser(BaseParser):
             headers=http_headers,
         )
         self.post_parser = PostsParser(verbose=verbose)
-        self.db = DatabaseProcessor(db_path=db_path)
+        self.db = self.__initial_db(format=format, db_path=db_path)
         self._db_initialized = False
         self.verbose = verbose
 
+    @staticmethod
+    def __initial_db(format: str, db_path: str):
+        if format == "sqlite":
+            return DatabaseProcessor(db_path=db_path)
+        else:
+            return CSVProcessor(csv_dir=db_path)
+
     @staticmethod
     def __forbidden_parse_preview(response: httpx.Response) -> bool:
         """

telegram_pm/run.py

@@ -27,6 +27,7 @@ class ParserRunner:
         self,
         db_path: str,
         channels: list[str],
+        format: str = "sqlite",
         verbose: bool = False,
         tg_before_param_size: int = config.TelegramConfig.before_param_size,
         tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
@@ -51,6 +52,7 @@ class ParserRunner:
         :param http_headers: HTTP headers
         """
         parser = PreviewParser(
+            format=format,
             channels=channels,
             verbose=verbose,
             db_path=db_path,
@@ -82,6 +84,7 @@ class ParserRunner:
 def run_tpm(
     db_path: str,
     channels: list[str],
+    format: str,
     verbose: bool = False,
     tg_before_param_size: int = config.TelegramConfig.before_param_size,
     tg_iteration_in_preview_count: int = config.TelegramConfig.iteration_in_preview_count,
@@ -95,6 +98,7 @@ def run_tpm(
     runner = ParserRunner()
     asyncio.run(
         runner.run(
+            format=format,
             channels=channels,
             verbose=verbose,
             db_path=db_path,

{telegram_pm-0.1.3.dist-info → telegram_pm-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: telegram-pm
-Version: 0.1.3
+Version: 0.1.5
 Summary: Telegram preview page parser
 Author: Your Name
 Author-email: you@example.com
@@ -37,7 +37,7 @@ Telegram monitoring tool for public channels that can be viewed via WEB preview.
 1. Ensure Python 3.12+ is installed (recommendation)
 2. Clone repository
 ```bash
-   git clone 'https://github.com/aIligat0r/tpm.git'
+git clone 'https://github.com/aIligat0r/tpm.git'
 ```
 or
 ```bash
@@ -77,13 +77,14 @@ poetry install
 
 #### 2. Launching the app
 
-| Options                           | Description                                                           | Required                                                       |
-|-----------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------|
-| `--db-path`                       | Path to the base (if not, it will be created)                         | ❌ required                                                     |
-| `--channels-filepath`/`--ch-file` | File of channel usernames (file where in each line Telegram username) | ❌ required (or usernames `--channel`/`--ch`)                   |
-| `--channel`/`--ch`                | List of usernames that are passed by the parameter                    | ❌ required (or file of channels `--channels-filepath`/`--chf`) |
-| `--verbose`/`--v`                 | Verbose mode                                                          | ➖                                                              |
-| `--help`/`--h`                    | Help information                                                      | ➖                                                              |
+| Options                       | Description                                                           | Required                                                       |
+|-------------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------|
+| `--db-path`                   | Path to db file (if sqlite). Else path to dir (if csv)                | ❌ required                                                     |
+| `--channels-filepath`/`--chf` | File of channel usernames (file where in each line Telegram username) | ❌ required (or usernames `--channel`/`--ch`)                   |
+| `--channel`/`--ch`            | List of usernames that are passed by the parameter                    | ❌ required (or file of channels `--channels-filepath`/`--chf`) |
+| `--verbose`/`--v`             | Verbose mode                                                          | ➖                                                              |
+| `--format`/`--f`              | Data saving format (csv, sqlite)                                      | ➖                                                              |
+| `--help`/`--h`                | Help information                                                      | ➖                                                              |
 
 **Poetry:**
 ```bash
@@ -107,7 +108,7 @@ $ chmod 666 ~/tpm_data_dir/telegram_messages.sqlite && chmod 666 ~/tpm_data_dir/
 docker run -it --rm \
     -v ~/tpm_data_dir/telegram_messages.sqlite:/data/telegram_messages.sqlite \
     -v ~/tpm_data_dir/usernames.txt:/data/usernames.txt \
-    telegram_pm --db-path /data/telegram_messages.sqlite --chf /data/usernames.txt
+    tpm --db-path /data/telegram_messages.sqlite --chf /data/usernames.txt
 ```
 **Python:**
 ```python
@@ -115,11 +116,12 @@ from telegram_pm.run import run_tpm
 
 
 run_tpm(
-    db_path="tg.db",                    # Path to sqlite database
+    db_path="tg.db",                    # Path to db file (if sqlite). Else path to dir (if csv)
     channels=["channel1", "channel2"],  # Channels list
     verbose=True,                       # Verbose mode
 
     # Configuration (optional)
+    format="sqlite",                    # Data saving format (csv, sqlite)
     tg_iteration_in_preview_count=5,    # Number of requests (default 5). 20 messages per request. (1 iter - last 20 messages)
     tg_sleep_time_seconds=60,           # Number of seconds after which the next process of receiving data from channels will begin (default 60 seconds)
     tg_sleep_after_error_request=30,    # Waiting after a failed requests (default 30)

{telegram_pm-0.1.3.dist-info → telegram_pm-0.1.5.dist-info}/RECORD

@@ -1,22 +1,23 @@
 telegram_pm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 telegram_pm/config.py,sha256=w1BZPxy8adyUnVQeGjUseSlVNRgpf7ZGXi4ltCXIo1Y,939
 telegram_pm/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-telegram_pm/database/db.py,sha256=rSfqCbBYrD4E_Msb5q8ilY1QIPlq7vnVE_-dNlYOXaM,4716
+telegram_pm/database/csv_processor.py,sha256=4qIHfwWuraUkvCePX68atF_34Zx0cOj9jwBZ1V3ACvU,3027
+telegram_pm/database/sqlite_processor.py,sha256=rSfqCbBYrD4E_Msb5q8ilY1QIPlq7vnVE_-dNlYOXaM,4716
 telegram_pm/entities.py,sha256=-mdx3u1M7bKFtEXaLcaaBjLQg08NBW77c2VeNHQQ_Gw,646
 telegram_pm/http_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 telegram_pm/http_client/client.py,sha256=EYFiCFZcICntF7Lc1QHsqQ_CcGtNI6G8j-DLmt1VJG4,1149
 telegram_pm/parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 telegram_pm/parsers/base.py,sha256=9GH7bJaqPueohRoK1OVOmjF9pY_fqaFizIc9Ak6qS-Y,22
 telegram_pm/parsers/post.py,sha256=4wf4KBG6NBOFGpk8_GH88M1hbyjTWXTQgRgGaHXgB40,10469
-telegram_pm/parsers/preview.py,sha256=TvWy36NOWvMMms3vUdc96wGRuYCvHI8R896gxiKrnJQ,7887
+telegram_pm/parsers/preview.py,sha256=axa2h0WdNObsRaXh2KYSS919r8Dq2KXeapopEQaV4uc,8224
 telegram_pm/parsers/tag_options.py,sha256=0YRQH5O8fpfReHRDXEThmFFyiacsUz-wlbjVFOLoiJ8,2040
-telegram_pm/run.py,sha256=dKMw2IrtCh3rkkyiwQHNQwRM97f887Z_LsKvUulomrI,4608
+telegram_pm/run.py,sha256=bGxXLWEHEaLRjnLWivGDXXX896JbwQRrqEjW4vd7oaM,4711
 telegram_pm/utils/__init__.py,sha256=loG7JOo8Th7vV7lYrVeCEhObguEaMQr7xRCmVkV7CM4,103
 telegram_pm/utils/logger.py,sha256=RqwcrFNMzjQfqB-aC9w79g9WLbcj6GvokRDtj9ZPH1Y,123
 telegram_pm/utils/parse.py,sha256=vSI4kNVvt2hqXLcOdp0MuCChG6fFqSrb17VzH6huqVQ,1167
 telegram_pm/utils/url.py,sha256=mv5Lc4PZbyL4hQXku3sGzMt3lmGKjtlYhbmzL0fKeb8,941
-telegram_pm-0.1.3.dist-info/entry_points.txt,sha256=dIvBN0V4aMrJKl7tB1qCYy7VM40uFqnuwcPibXfnSU0,40
-telegram_pm-0.1.3.dist-info/LICENSE,sha256=kaLyGzbJPljgIIJrGiWc2611z1YfjYG8QsI6v0C_oug,1066
-telegram_pm-0.1.3.dist-info/METADATA,sha256=oBdSMbKZlzMKzn3i8jr1WmzwAUT4rzaxWTQwLMOn1cw,8402
-telegram_pm-0.1.3.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-telegram_pm-0.1.3.dist-info/RECORD,,
+telegram_pm-0.1.5.dist-info/entry_points.txt,sha256=ZEANlDcQc97SLwP-sAJ7kdWHX8is8Q2wIbJJC3s-cGA,48
+telegram_pm-0.1.5.dist-info/LICENSE,sha256=kaLyGzbJPljgIIJrGiWc2611z1YfjYG8QsI6v0C_oug,1066
+telegram_pm-0.1.5.dist-info/METADATA,sha256=ZlusZgwQAWKAOoPHNsHUW10SI3_cB2baREmAo9AGDyA,8642
+telegram_pm-0.1.5.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+telegram_pm-0.1.5.dist-info/RECORD,,

telegram_pm-0.1.5.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+telegram-pm=commands.cli:app
+

telegram_pm-0.1.3.dist-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-tpm=commands.cli:app
-