tgdata 0.0.0__tar.gz → 0.0.1__tar.gz

{tgdata-0.0.0 → tgdata-0.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tgdata
-Version: 0.0.0
+Version: 0.0.1
 Summary: A production-grade Python library for extracting and processing Telegram group and channel messages
 Author: enes kuzucu
 Classifier: Development Status :: 3 - Alpha
@@ -46,20 +46,72 @@ A production-grade Python library for extracting and processing Telegram group a
 pip install tgdata
 ```
 
+
+## Authentication
+
+tgdata supports 1 authentication at the moment. 
+
+(2 other are being implemented)
+
+1. get telegram development credentials in telegram API Development Tools from [https://my.telegram.org/apps](https://my.telegram.org/apps)
+
+2. Create a `config.ini` file with your Telegram API credentials like this:
+
+```ini
+[Telegram]
+
+# you can get telegram development credentials in telegram API Development Tools
+api_id = 1234566
+api_hash = a24adjfakjdfakjshdflkajsbdflk
+phone = +905064004949 # use full phone number including + and country code
+username = 'powerpuffdude'
+
+```
+
+
 ## Quick Start
 
+### List All Group Chats 
+
+This is required for finding chat id for the chat of interest. 
+
 ```python
 from tgdata import TgData
 import asyncio
 
 async def main():
     # Initialize the client
-    tg = TgData()
+    tg = TgData("config.ini")
     
     # List available groups and channels
     groups = await tg.list_groups()
-    print(f"Found {len(groups)} groups/channels")
-    
+
+    print(groups)
+
+```
+
+Outputs:
+
+```
+Python Devs 10012312313
+```
+
+
+### Get all messages of a chat 
+
+Once you optain chat id, you can use it to get all messages. 
+Upto 2500 messages you will not see limiting from Telegram. 
+If group is really active (+20 messages each day for over a year) then it is better to use batch logic. 
+
+
+```python
+from tgdata import TgData
+import asyncio
+
+async def main():
+    # Initialize the client
+    tg = TgData("config.ini")
+
     # Fetch messages from a specific group
     messages = await tg.get_messages(
         group_id=-1001234567890,  # Your group ID
@@ -69,40 +121,43 @@ async def main():
     
     # Export to CSV
     tg.export_messages(messages, "messages.csv")
-    
-    # Get message statistics
-    stats = tg.get_statistics(messages)
-    print(f"Total messages: {stats['total_messages']}")
-    print(f"Date range: {stats['date_range']['first']} to {stats['date_range']['last']}")
 
 asyncio.run(main())
 ```
 
-## Configuration
+### Get message count of a chat 
 
-Create a `config.ini` file with your Telegram API credentials:
 
-```ini
-[telegram]
-api_id = YOUR_API_ID
-api_hash = YOUR_API_HASH
-session_file = telegram_session
+
+```python
+from tgdata import TgData
+import asyncio
+
+async def main():
+    # Initialize the client
+    tg = TgData("config.ini")
+
+    # Fetch messages from a specific group
+    message_count = await tg.get_message_count(
+        group_id=-1001234567890,  # Your group ID
+      
+    )
+
+    print(message_count)
+
+asyncio.run(main())
 ```
 
-Get your API credentials from [https://my.telegram.org/apps](https://my.telegram.org/apps)
+
+
 
 ## Advanced Usage
 
-### ETL Pipeline Integration
+### get_messages with start_date parameter
 
 ```python
-from tgdata import TgData
-from datetime import datetime, timedelta
-import asyncio
 
-async def etl_pipeline():
-    tg = TgData()
-    
+    tg = TgData("config.ini")
     # Fetch recent messages for ETL processing
     yesterday = datetime.now() - timedelta(days=1)
     messages = await tg.get_messages(
@@ -111,21 +166,7 @@ async def etl_pipeline():
         with_progress=True
     )
     
-    # Filter messages
-    filtered = tg.filter_messages(
-        messages,
-        keyword="important",
-        start_date=yesterday
-    )
-    
-    # Export for data pipeline
-    tg.export_messages(filtered, "daily_extract.json", format="json")
-    
-    # Get metrics
-    metrics = await tg.get_metrics()
-    print(f"Processed {len(filtered)} messages")
 
-asyncio.run(etl_pipeline())
 ```
 
 ### Incremental Message Fetching
@@ -134,9 +175,9 @@ asyncio.run(etl_pipeline())
 from tgdata import TgData
 
 async def incremental_fetch():
-    tg = TgData()
+    tg = TgData("config.ini")
     
-    # Get the latest message ID from your storage
+    # Get the latest message ID from your storage, or db
     last_processed_id = load_checkpoint()  # Your implementation
     
     # Fetch only messages after that ID
@@ -144,15 +185,9 @@ async def incremental_fetch():
         group_id=-1001234567890,
         after_id=last_processed_id
     )
-    
-    if not new_messages.empty:
-        # Process new messages
-        process_messages(new_messages)
-        
-        # Save new checkpoint
-        save_checkpoint(new_messages['MessageId'].max())
-    
-    print(f"New messages: {len(new_messages)}")
+
+    save_checkpoint(new_messages['MessageId'].max())   # Your implementation
+
 
 asyncio.run(incremental_fetch())
 ```
@@ -174,24 +209,10 @@ async def monitor_extraction():
     )
 ```
 
-### Real-time Message Monitoring
+### Custom callback 
+
+
 
-```python
-async def monitor_real_time():
-    tg = TgData()
-    
-    # Register handler for new messages
-    @tg.on_new_message(group_id=-1001234567890)
-    async def handle_message(event):
-        print(f"New message from {event.sender_id}: {event.message.text}")
-        
-        # React to commands
-        if event.message.text == "!ping":
-            await event.reply("Pong!")
-    
-    # Run event loop
-    await tg.run_with_event_loop()
-```
 
 ### Polling for New Messages
 
@@ -213,69 +234,9 @@ async def poll_messages():
     )
 ```
 
-## API Reference
-
-### TgData
-
-Main class for interacting with Telegram groups and channels.
-
-#### Methods
-
-**Core Methods:**
-- `list_groups()` - List all accessible groups and channels
-- `get_messages()` - Fetch messages with various filters
-- `search_messages()` - Search for specific content
-- `filter_messages()` - Filter messages by criteria
-- `export_messages()` - Export to CSV or JSON
-- `get_statistics()` - Get message statistics
-- `get_metrics()` - Get session metrics
-- `get_message_count()` - Get total message count without fetching all
 
-**Real-time & Polling:**
-- `on_new_message()` - Decorator to register real-time message handlers
-- `poll_for_messages()` - Poll for new messages at intervals
-- `run_with_event_loop()` - Run client with event loop for real-time events
+###  on_new_message method
 
-### Checkpoint Management
-
-For production ETL pipelines, implement your own checkpoint logic:
-
-```python
-# Save last processed message ID to your database
-last_id = messages['MessageId'].max()
-await save_to_database(group_id, last_id)
-
-# On next run, fetch only new messages
-last_id = await load_from_database(group_id)
-new_messages = await tg.get_messages(group_id=group_id, after_id=last_id)
-```
-
-## Production Deployment
-
-### Best Practices
-
-1. **Use environment variables for credentials**:
-   ```python
-   import os
-   config_path = os.getenv("TELEGRAM_CONFIG", "config.ini")
-   tg = TgData(config_path=config_path)
-   ```
-
-2. **Implement error handling**:
-   ```python
-   try:
-       messages = await tg.get_messages(group_id=group_id)
-   except Exception as e:
-       logger.error(f"Failed to fetch messages: {e}")
-       # Implement retry logic or alerting
-   ```
-
-3. **Monitor rate limits**:
-   ```python
-   health = await tg.health_check()
-   if not health['primary_connection']:
-       # Handle connection issues
-   ```
 
 ### Performance Tips
 
@@ -297,13 +258,3 @@ MIT License - see LICENSE file for details
 ## Contributing
 
 Contributions are welcome! Please feel free to submit a Pull Request.
-
-## Support
-
-- Documentation: [Full documentation](https://github.com/yourusername/tgdata)
-- Issues: [GitHub Issues](https://github.com/yourusername/tgdata/issues)
-- Examples: See the [examples/](examples/) directory
-
-## Disclaimer
-
-This tool is for legitimate data collection and analysis only. Users must comply with Telegram's Terms of Service and applicable laws. Always respect user privacy and platform rate limits.

tgdata-0.0.1/README.md

@@ -0,0 +1,234 @@
+# tgdata
+
+A production-grade Python library for extracting and processing Telegram group and channel messages. Designed for ETL pipelines, data analysis, and archival purposes.
+
+## Features
+
+- 🚀 **Production-Ready**: Built for reliability and scale in ETL pipelines
+- 📊 **Efficient Data Extraction**: Fetch messages from groups and channels with rate limit handling
+- 🔄 **Incremental Updates**: Fetch only new messages with `after_id` parameter
+- 📈 **Progress Tracking**: Monitor long-running operations with real-time progress
+- 🔌 **Clean Architecture**: Focused on data extraction with minimal dependencies
+- 🛡️ **Robust Error Handling**: Automatic retries with exponential backoff
+- 📁 **Multiple Export Formats**: Export to CSV, JSON, or integrate with your data pipeline
+- 🔔 **Real-time Updates**: Listen for new messages with event handlers
+- ⏱️ **Polling Support**: Poll for new messages at configurable intervals
+
+## Installation
+
+```bash
+pip install tgdata
+```
+
+
+## Authentication
+
+tgdata supports 1 authentication at the moment. 
+
+(2 other are being implemented)
+
+1. get telegram development credentials in telegram API Development Tools from [https://my.telegram.org/apps](https://my.telegram.org/apps)
+
+2. Create a `config.ini` file with your Telegram API credentials like this:
+
+```ini
+[Telegram]
+
+# you can get telegram development credentials in telegram API Development Tools
+api_id = 1234566
+api_hash = a24adjfakjdfakjshdflkajsbdflk
+phone = +905064004949 # use full phone number including + and country code
+username = 'powerpuffdude'
+
+```
+
+
+## Quick Start
+
+### List All Group Chats 
+
+This is required for finding chat id for the chat of interest. 
+
+```python
+from tgdata import TgData
+import asyncio
+
+async def main():
+    # Initialize the client
+    tg = TgData("config.ini")
+    
+    # List available groups and channels
+    groups = await tg.list_groups()
+
+    print(groups)
+
+```
+
+Outputs:
+
+```
+Python Devs 10012312313
+```
+
+
+### Get all messages of a chat 
+
+Once you optain chat id, you can use it to get all messages. 
+Upto 2500 messages you will not see limiting from Telegram. 
+If group is really active (+20 messages each day for over a year) then it is better to use batch logic. 
+
+
+```python
+from tgdata import TgData
+import asyncio
+
+async def main():
+    # Initialize the client
+    tg = TgData("config.ini")
+
+    # Fetch messages from a specific group
+    messages = await tg.get_messages(
+        group_id=-1001234567890,  # Your group ID
+        limit=1000,
+        with_progress=True
+    )
+    
+    # Export to CSV
+    tg.export_messages(messages, "messages.csv")
+
+asyncio.run(main())
+```
+
+### Get message count of a chat 
+
+
+
+```python
+from tgdata import TgData
+import asyncio
+
+async def main():
+    # Initialize the client
+    tg = TgData("config.ini")
+
+    # Fetch messages from a specific group
+    message_count = await tg.get_message_count(
+        group_id=-1001234567890,  # Your group ID
+      
+    )
+
+    print(message_count)
+
+asyncio.run(main())
+```
+
+
+
+
+## Advanced Usage
+
+### get_messages with start_date parameter
+
+```python
+
+    tg = TgData("config.ini")
+    # Fetch recent messages for ETL processing
+    yesterday = datetime.now() - timedelta(days=1)
+    messages = await tg.get_messages(
+        group_id=-1001234567890,
+        start_date=yesterday,
+        with_progress=True
+    )
+    
+
+```
+
+### Incremental Message Fetching
+
+```python
+from tgdata import TgData
+
+async def incremental_fetch():
+    tg = TgData("config.ini")
+    
+    # Get the latest message ID from your storage, or db
+    last_processed_id = load_checkpoint()  # Your implementation
+    
+    # Fetch only messages after that ID
+    new_messages = await tg.get_messages(
+        group_id=-1001234567890,
+        after_id=last_processed_id
+    )
+
+    save_checkpoint(new_messages['MessageId'].max())   # Your implementation
+
+
+asyncio.run(incremental_fetch())
+```
+
+### Progress Monitoring
+
+```python
+async def monitor_extraction():
+    tg = TgData()
+    
+    def progress_callback(current, total, rate):
+        percent = (current / total * 100) if total else 0
+        print(f"Progress: {current}/{total} ({percent:.1f}%) - {rate:.1f} msg/s")
+    
+    messages = await tg.get_messages(
+        group_id=-1001234567890,
+        limit=10000,
+        progress_callback=progress_callback
+    )
+```
+
+### Custom callback 
+
+
+
+
+### Polling for New Messages
+
+```python
+async def poll_messages():
+    tg = TgData()
+    
+    # Define callback for new messages
+    async def process_batch(messages_df):
+        print(f"Got {len(messages_df)} new messages")
+        # Process messages here
+    
+    # Poll every 30 seconds
+    await tg.poll_for_messages(
+        group_id=-1001234567890,
+        interval=30,
+        callback=process_batch,
+        max_iterations=10  # Stop after 10 polls
+    )
+```
+
+
+###  on_new_message method
+
+
+### Performance Tips
+
+- Use connection pooling for parallel operations
+- Implement checkpoint logic for incremental processing
+- Implement progress callbacks for visibility
+- Export data incrementally for large datasets
+
+## Requirements
+
+- Python 3.7+
+- Telegram API credentials (not bot tokens)
+- Group/channel membership
+
+## License
+
+MIT License - see LICENSE file for details
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.

{tgdata-0.0.0 → tgdata-0.0.1}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='tgdata',  # Package name
-    version='0.0.0',  # Version of your package
+    version='0.0.1',  # Version of your package
     author='enes kuzucu',  # Your name
     
     description='A production-grade Python library for extracting and processing Telegram group and channel messages', 

{tgdata-0.0.0 → tgdata-0.0.1}/tgdata/connection_engine.py

@@ -294,10 +294,26 @@ class ConnectionEngine:
             logger.error(f"Connection validation failed: {e}")
             return False
             
-    async def handle_rate_limit(self, error: FloodWaitError, client: Optional[TelegramClient] = None):
-        """Handle rate limit errors"""
-        wait_time = error.seconds
-        logger.warning(f"Rate limit hit! Waiting {wait_time} seconds...")
+    async def handle_rate_limit(self, error: FloodWaitError, client: Optional[TelegramClient] = None, strategy: str = 'wait'):
+        """
+        Handle rate limit errors with configurable strategy.
+        
+        Args:
+            error: The FloodWaitError from Telegram
+            client: The client that hit the rate limit
+            strategy: 'wait' (wait exact time) or 'exponential' (exponential backoff)
+        """
+        base_wait_time = error.seconds
+        
+        if strategy == 'exponential':
+            # Add random jitter (0-30% extra) to prevent thundering herd
+            import random
+            jitter = random.uniform(0, 0.3)
+            wait_time = base_wait_time * (1 + jitter)
+            logger.warning(f"Rate limit hit! Using exponential backoff: waiting {wait_time:.1f}s (base: {base_wait_time}s)")
+        else:
+            wait_time = base_wait_time
+            logger.warning(f"Rate limit hit! Waiting {wait_time} seconds...")
         
         if self._pool and client:
             self._pool.mark_rate_limited(client, wait_time)