tgdata 0.0.0__py3-none-any.whl

tgdata/utils.py

@@ -0,0 +1,247 @@
+"""
+Utility functions for the Telegram Group Message Crawler.
+"""
+
+import json
+import os
+from datetime import datetime
+from typing import Any, Dict, Optional
+import pandas as pd
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def format_message_for_display(message: Dict[str, Any], max_length: int = 100) -> str:
+    """
+    Format a message for display.
+    
+    Args:
+        message: Message dictionary
+        max_length: Maximum message length to display
+        
+    Returns:
+        Formatted message string
+    """
+    msg_text = message.get('Message', '')
+    if msg_text and len(msg_text) > max_length:
+        msg_text = f"{msg_text[:max_length]}..."
+        
+    return (
+        f"{message.get('Date', 'Unknown date')} | "
+        f"{message.get('Name', 'Unknown')} "
+        f"({message.get('Username', 'No username')}): "
+        f"{msg_text or '[No text]'}"
+    )
+
+
+def export_to_json(df: pd.DataFrame, filepath: str, pretty: bool = True) -> None:
+    """
+    Export DataFrame to JSON file.
+    
+    Args:
+        df: DataFrame to export
+        filepath: Output file path
+        pretty: Whether to pretty-print JSON
+    """
+    try:
+        # Convert DataFrame to dict, handling datetime objects
+        data = df.to_dict('records')
+        
+        # Convert datetime objects to strings
+        for record in data:
+            for key, value in record.items():
+                if isinstance(value, datetime):
+                    record[key] = value.isoformat()
+                elif isinstance(value, bytes):
+                    # Don't include binary data in JSON
+                    record[key] = "[Binary data]"
+                    
+        # Write to file
+        with open(filepath, 'w', encoding='utf-8') as f:
+            if pretty:
+                json.dump(data, f, indent=2, ensure_ascii=False)
+            else:
+                json.dump(data, f, ensure_ascii=False)
+                
+        logger.info(f"Exported {len(df)} messages to {filepath}")
+        
+    except Exception as e:
+        logger.error(f"Error exporting to JSON: {e}")
+        raise
+
+
+def export_to_csv(df: pd.DataFrame, filepath: str) -> None:
+    """
+    Export DataFrame to CSV file.
+    
+    Args:
+        df: DataFrame to export
+        filepath: Output file path
+    """
+    try:
+        # Remove binary data columns for CSV export
+        export_df = df.copy()
+        if 'PhotoData' in export_df.columns:
+            export_df = export_df.drop('PhotoData', axis=1)
+            
+        export_df.to_csv(filepath, index=False, encoding='utf-8')
+        logger.info(f"Exported {len(df)} messages to {filepath}")
+        
+    except Exception as e:
+        logger.error(f"Error exporting to CSV: {e}")
+        raise
+
+
+def filter_messages_by_sender(df: pd.DataFrame, sender_id: Optional[int] = None, 
+                            username: Optional[str] = None) -> pd.DataFrame:
+    """
+    Filter messages by sender.
+    
+    Args:
+        df: Messages DataFrame
+        sender_id: Sender ID to filter by
+        username: Username to filter by
+        
+    Returns:
+        Filtered DataFrame
+    """
+    if sender_id:
+        df = df[df['SenderId'] == sender_id]
+    elif username:
+        df = df[df['Username'].str.contains(username, case=False, na=False)]
+        
+    return df
+
+
+def filter_messages_by_content(df: pd.DataFrame, keyword: str, 
+                             case_sensitive: bool = False) -> pd.DataFrame:
+    """
+    Filter messages containing specific keyword.
+    
+    Args:
+        df: Messages DataFrame
+        keyword: Keyword to search for
+        case_sensitive: Whether search is case sensitive
+        
+    Returns:
+        Filtered DataFrame
+    """
+    return df[df['Message'].str.contains(
+        keyword, 
+        case=case_sensitive, 
+        na=False
+    )]
+
+
+def get_message_statistics(df: pd.DataFrame) -> Dict[str, Any]:
+    """
+    Get statistics about messages.
+    
+    Args:
+        df: Messages DataFrame
+        
+    Returns:
+        Dictionary with statistics
+    """
+    if df.empty:
+        return {
+            'total_messages': 0,
+            'unique_senders': 0,
+            'date_range': None,
+            'top_senders': []
+        }
+        
+    stats = {
+        'total_messages': len(df),
+        'unique_senders': df['SenderId'].nunique(),
+        'date_range': {
+            'start': df['Date'].min(),
+            'end': df['Date'].max()
+        },
+        'messages_with_text': df['Message'].notna().sum(),
+        'replies': df['ReplyToId'].notna().sum(),
+        'forwards': df['ForwardedFrom'].notna().sum()
+    }
+    
+    # Top senders
+    top_senders = df.groupby(['SenderId', 'Name']).size().sort_values(
+        ascending=False
+    ).head(10)
+    
+    stats['top_senders'] = [
+        {
+            'sender_id': sender_id,
+            'name': name,
+            'message_count': count
+        }
+        for (sender_id, name), count in top_senders.items()
+    ]
+    
+    return stats
+
+
+def save_profile_photos(df: pd.DataFrame, output_dir: str) -> None:
+    """
+    Save profile photos from messages to directory.
+    
+    Args:
+        df: Messages DataFrame with PhotoData column
+        output_dir: Directory to save photos
+    """
+    if 'PhotoData' not in df.columns:
+        logger.warning("No PhotoData column in DataFrame")
+        return
+        
+    os.makedirs(output_dir, exist_ok=True)
+    saved_count = 0
+    
+    for _, row in df.iterrows():
+        if row['PhotoData'] is not None:
+            filename = f"{row['SenderId']}_{row['Username'].replace('@', '')}.jpg"
+            filepath = os.path.join(output_dir, filename)
+            
+            try:
+                with open(filepath, 'wb') as f:
+                    f.write(row['PhotoData'])
+                saved_count += 1
+            except Exception as e:
+                logger.error(f"Error saving photo for {row['SenderId']}: {e}")
+                
+    logger.info(f"Saved {saved_count} profile photos to {output_dir}")
+
+
+def create_metrics_report(df: pd.DataFrame, group_info: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Create a comprehensive metrics report.
+    
+    Args:
+        df: Messages DataFrame
+        group_info: Information about the group
+        
+    Returns:
+        Dictionary with metrics
+    """
+    stats = get_message_statistics(df)
+    
+    report = {
+        'timestamp': datetime.now().isoformat(),
+        'group': group_info,
+        'statistics': stats,
+        'processing_info': {
+            'total_processed': len(df),
+            'memory_usage_mb': df.memory_usage(deep=True).sum() / 1024 / 1024
+        }
+    }
+    
+    # Add time-based statistics if we have data
+    if not df.empty:
+        df['Hour'] = pd.to_datetime(df['Date']).dt.hour
+        df['DayOfWeek'] = pd.to_datetime(df['Date']).dt.day_name()
+        
+        report['time_analysis'] = {
+            'messages_by_hour': df['Hour'].value_counts().sort_index().to_dict(),
+            'messages_by_day': df['DayOfWeek'].value_counts().to_dict()
+        }
+        
+    return report

tgdata-0.0.0.dist-info/METADATA

@@ -0,0 +1,309 @@
+Metadata-Version: 2.4
+Name: tgdata
+Version: 0.0.0
+Summary: A production-grade Python library for extracting and processing Telegram group and channel messages
+Author: enes kuzucu
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: Telethon
+Requires-Dist: pandas
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# tgdata
+
+A production-grade Python library for extracting and processing Telegram group and channel messages. Designed for ETL pipelines, data analysis, and archival purposes.
+
+## Features
+
+- 🚀 **Production-Ready**: Built for reliability and scale in ETL pipelines
+- 📊 **Efficient Data Extraction**: Fetch messages from groups and channels with rate limit handling
+- 🔄 **Incremental Updates**: Fetch only new messages with `after_id` parameter
+- 📈 **Progress Tracking**: Monitor long-running operations with real-time progress
+- 🔌 **Clean Architecture**: Focused on data extraction with minimal dependencies
+- 🛡️ **Robust Error Handling**: Automatic retries with exponential backoff
+- 📁 **Multiple Export Formats**: Export to CSV, JSON, or integrate with your data pipeline
+- 🔔 **Real-time Updates**: Listen for new messages with event handlers
+- ⏱️ **Polling Support**: Poll for new messages at configurable intervals
+
+## Installation
+
+```bash
+pip install tgdata
+```
+
+## Quick Start
+
+```python
+from tgdata import TgData
+import asyncio
+
+async def main():
+    # Initialize the client
+    tg = TgData()
+    
+    # List available groups and channels
+    groups = await tg.list_groups()
+    print(f"Found {len(groups)} groups/channels")
+    
+    # Fetch messages from a specific group
+    messages = await tg.get_messages(
+        group_id=-1001234567890,  # Your group ID
+        limit=1000,
+        with_progress=True
+    )
+    
+    # Export to CSV
+    tg.export_messages(messages, "messages.csv")
+    
+    # Get message statistics
+    stats = tg.get_statistics(messages)
+    print(f"Total messages: {stats['total_messages']}")
+    print(f"Date range: {stats['date_range']['first']} to {stats['date_range']['last']}")
+
+asyncio.run(main())
+```
+
+## Configuration
+
+Create a `config.ini` file with your Telegram API credentials:
+
+```ini
+[telegram]
+api_id = YOUR_API_ID
+api_hash = YOUR_API_HASH
+session_file = telegram_session
+```
+
+Get your API credentials from [https://my.telegram.org/apps](https://my.telegram.org/apps)
+
+## Advanced Usage
+
+### ETL Pipeline Integration
+
+```python
+from tgdata import TgData
+from datetime import datetime, timedelta
+import asyncio
+
+async def etl_pipeline():
+    tg = TgData()
+    
+    # Fetch recent messages for ETL processing
+    yesterday = datetime.now() - timedelta(days=1)
+    messages = await tg.get_messages(
+        group_id=-1001234567890,
+        start_date=yesterday,
+        with_progress=True
+    )
+    
+    # Filter messages
+    filtered = tg.filter_messages(
+        messages,
+        keyword="important",
+        start_date=yesterday
+    )
+    
+    # Export for data pipeline
+    tg.export_messages(filtered, "daily_extract.json", format="json")
+    
+    # Get metrics
+    metrics = await tg.get_metrics()
+    print(f"Processed {len(filtered)} messages")
+
+asyncio.run(etl_pipeline())
+```
+
+### Incremental Message Fetching
+
+```python
+from tgdata import TgData
+
+async def incremental_fetch():
+    tg = TgData()
+    
+    # Get the latest message ID from your storage
+    last_processed_id = load_checkpoint()  # Your implementation
+    
+    # Fetch only messages after that ID
+    new_messages = await tg.get_messages(
+        group_id=-1001234567890,
+        after_id=last_processed_id
+    )
+    
+    if not new_messages.empty:
+        # Process new messages
+        process_messages(new_messages)
+        
+        # Save new checkpoint
+        save_checkpoint(new_messages['MessageId'].max())
+    
+    print(f"New messages: {len(new_messages)}")
+
+asyncio.run(incremental_fetch())
+```
+
+### Progress Monitoring
+
+```python
+async def monitor_extraction():
+    tg = TgData()
+    
+    def progress_callback(current, total, rate):
+        percent = (current / total * 100) if total else 0
+        print(f"Progress: {current}/{total} ({percent:.1f}%) - {rate:.1f} msg/s")
+    
+    messages = await tg.get_messages(
+        group_id=-1001234567890,
+        limit=10000,
+        progress_callback=progress_callback
+    )
+```
+
+### Real-time Message Monitoring
+
+```python
+async def monitor_real_time():
+    tg = TgData()
+    
+    # Register handler for new messages
+    @tg.on_new_message(group_id=-1001234567890)
+    async def handle_message(event):
+        print(f"New message from {event.sender_id}: {event.message.text}")
+        
+        # React to commands
+        if event.message.text == "!ping":
+            await event.reply("Pong!")
+    
+    # Run event loop
+    await tg.run_with_event_loop()
+```
+
+### Polling for New Messages
+
+```python
+async def poll_messages():
+    tg = TgData()
+    
+    # Define callback for new messages
+    async def process_batch(messages_df):
+        print(f"Got {len(messages_df)} new messages")
+        # Process messages here
+    
+    # Poll every 30 seconds
+    await tg.poll_for_messages(
+        group_id=-1001234567890,
+        interval=30,
+        callback=process_batch,
+        max_iterations=10  # Stop after 10 polls
+    )
+```
+
+## API Reference
+
+### TgData
+
+Main class for interacting with Telegram groups and channels.
+
+#### Methods
+
+**Core Methods:**
+- `list_groups()` - List all accessible groups and channels
+- `get_messages()` - Fetch messages with various filters
+- `search_messages()` - Search for specific content
+- `filter_messages()` - Filter messages by criteria
+- `export_messages()` - Export to CSV or JSON
+- `get_statistics()` - Get message statistics
+- `get_metrics()` - Get session metrics
+- `get_message_count()` - Get total message count without fetching all
+
+**Real-time & Polling:**
+- `on_new_message()` - Decorator to register real-time message handlers
+- `poll_for_messages()` - Poll for new messages at intervals
+- `run_with_event_loop()` - Run client with event loop for real-time events
+
+### Checkpoint Management
+
+For production ETL pipelines, implement your own checkpoint logic:
+
+```python
+# Save last processed message ID to your database
+last_id = messages['MessageId'].max()
+await save_to_database(group_id, last_id)
+
+# On next run, fetch only new messages
+last_id = await load_from_database(group_id)
+new_messages = await tg.get_messages(group_id=group_id, after_id=last_id)
+```
+
+## Production Deployment
+
+### Best Practices
+
+1. **Use environment variables for credentials**:
+   ```python
+   import os
+   config_path = os.getenv("TELEGRAM_CONFIG", "config.ini")
+   tg = TgData(config_path=config_path)
+   ```
+
+2. **Implement error handling**:
+   ```python
+   try:
+       messages = await tg.get_messages(group_id=group_id)
+   except Exception as e:
+       logger.error(f"Failed to fetch messages: {e}")
+       # Implement retry logic or alerting
+   ```
+
+3. **Monitor rate limits**:
+   ```python
+   health = await tg.health_check()
+   if not health['primary_connection']:
+       # Handle connection issues
+   ```
+
+### Performance Tips
+
+- Use connection pooling for parallel operations
+- Implement checkpoint logic for incremental processing
+- Implement progress callbacks for visibility
+- Export data incrementally for large datasets
+
+## Requirements
+
+- Python 3.7+
+- Telegram API credentials (not bot tokens)
+- Group/channel membership
+
+## License
+
+MIT License - see LICENSE file for details
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## Support
+
+- Documentation: [Full documentation](https://github.com/yourusername/tgdata)
+- Issues: [GitHub Issues](https://github.com/yourusername/tgdata/issues)
+- Examples: See the [examples/](examples/) directory
+
+## Disclaimer
+
+This tool is for legitimate data collection and analysis only. Users must comply with Telegram's Terms of Service and applicable laws. Always respect user privacy and platform rate limits.

tgdata-0.0.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+tgdata/__init__.py,sha256=fayYjaDv4oV5z09koG0ieBhxXCZyyXou3lLBonh3fuc,1067
+tgdata/connection_engine.py,sha256=-jegoNPZ4Wjt24hLyY3OoZ4L38faMLaUiFXPlrBdS9A,11796
+tgdata/message_engine.py,sha256=aMc_EptGMWAxyd1JNxHofJkSt4BvB94DdeqzJt5wyIk,11135
+tgdata/models.py,sha256=mONAsXXqiklF-Q7q9_JuvDHy_lODeQuCuwR9j5As0Yg,1702
+tgdata/progress.py,sha256=oQOhiPWAwo4zBgVDqrYyIeoCFG-WYx_xo7Auwi9O324,3382
+tgdata/tgdata.py,sha256=WGaViROT99JUFhRyInbjs02k4-Keu_YOQ5u_Pc-lzrY,19462
+tgdata/utils.py,sha256=LDGZ8KluJ0tEuL7S5F08ln75g6UWxOL7lRnn7jZ9p68,7035
+tgdata/smoke_tests/__init__.py,sha256=7DrB-Ju2fq9CURHEawNC7tqym3GAWCCvWdhelUgrXL8,48
+tgdata/smoke_tests/my_test.py,sha256=GaZ29JEJrd42-lgpA9FjLkWNC8rR0HLywYSEGNKWpOQ,5484
+tgdata/smoke_tests/test_00_connection.py,sha256=NB_cN0X8CtCSjT9vRz4I3XjM5ZbiJ9K4lnfRaw7GXLw,9332
+tgdata/smoke_tests/test_01_list_group_chats.py,sha256=XFh2ugY44W7VBJbZcyBpB4q7K7i7INKp2HEkTEmrEnM,15639
+tgdata/smoke_tests/test_02_get_message_count.py,sha256=n3fhx_iKFdXT1YXbfDeCSIGuS_Ply0AY1XS1y7t46L4,1990
+tgdata/smoke_tests/test_03_get_all_messages.py,sha256=hNfdSxVaqCeCNMf03cEjmjeNZlFG8KU52BYMWfTVWFs,1739
+tgdata/smoke_tests/test_04_get_all_messages_in_batches.py,sha256=Pl4gMLjZAe_TPaCKiI4wyNaQcHKkW9ZC1aWswHuNQe8,2890
+tgdata/smoke_tests/test_06_advanced_features.py,sha256=XC_CaZ1FB1mzq2o7pUS5SaZZ8wDRtGd7fY6lzWJTmBM,7557
+tgdata/smoke_tests/test_10_polling.py,sha256=GGqjdK6p0_JOt2iAepw03Fc7NB8YSNrixvl_-bkt2s0,7490
+tgdata-0.0.0.dist-info/METADATA,sha256=asaNYdPxQKHmKCrmmPaPQtJL082wFCilKUcDz4i_FHE,8526
+tgdata-0.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+tgdata-0.0.0.dist-info/top_level.txt,sha256=mvOH6wBLbA5K_TUruSVtC4r9r7eoQ9qdCytRE4DJWAI,7
+tgdata-0.0.0.dist-info/RECORD,,

tgdata-0.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tgdata-0.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+tgdata