tgdata 0.0.0__py3-none-any.whl

tgdata/models.py

@@ -0,0 +1,69 @@
+"""
+Data models for the Telegram Group Message Crawler.
+"""
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional, Any, Dict
+
+
+
+@dataclass
+class RateLimitInfo:
+    """Track rate limit information"""
+    requests_made: int = 0
+    window_start: float = 0
+    last_request: float = 0
+    flood_wait_until: float = 0
+
+
+@dataclass
+class GroupInfo:
+    """Information about a Telegram group/channel"""
+    id: int
+    title: str
+    username: Optional[str] = None
+    is_channel: bool = False
+    is_megagroup: bool = False
+    participants_count: Optional[int] = None
+
+
+@dataclass
+class MessageData:
+    """Structured message data"""
+    message_id: int
+    sender_id: int
+    sender_name: str
+    username: str
+    message: Optional[str]
+    date: datetime
+    reply_to_id: Optional[int] = None
+    forwarded_from: Optional[int] = None
+    photo_data: Optional[bytes] = None
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for DataFrame creation"""
+        return {
+            'MessageId': self.message_id,
+            'SenderId': self.sender_id,
+            'Name': self.sender_name,
+            'Username': self.username,
+            'Message': self.message,
+            'Date': self.date,
+            'ReplyToId': self.reply_to_id,
+            'ForwardedFrom': self.forwarded_from,
+            'PhotoData': self.photo_data
+        }
+
+
+@dataclass
+class ConnectionConfig:
+    """Configuration for Telegram connection"""
+    api_id: str
+    api_hash: str
+    session_file: str
+    phone: Optional[str] = None
+    username: Optional[str] = None
+    max_retries: int = 3
+    retry_delay: float = 1.0
+    exponential_backoff: bool = True

tgdata/progress.py

@@ -0,0 +1,105 @@
+"""
+Progress tracking utilities for long-running operations.
+"""
+
+from datetime import datetime, timedelta
+from typing import Optional, Callable
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ProgressTracker:
+    """
+    Track progress of long-running operations.
+    This doesn't need persistence as it's for real-time progress only.
+    """
+    
+    def __init__(self, 
+                 total_expected: Optional[int] = None,
+                 callback: Optional[Callable[[int, Optional[int], float], None]] = None):
+        """
+        Initialize progress tracker.
+        
+        Args:
+            total_expected: Expected total items (if known)
+            callback: Optional callback function(current, total, rate)
+        """
+        self.total_expected = total_expected
+        self.callback = callback
+        self.current = 0
+        self.start_time: Optional[datetime] = None
+        
+    def start(self) -> None:
+        """Start tracking progress."""
+        self.start_time = datetime.now()
+        self.current = 0
+        logger.debug("Progress tracking started")
+        
+    def update(self, count: int = 1) -> None:
+        """
+        Update progress.
+        
+        Args:
+            count: Number of items processed
+        """
+        self.current += count
+        
+        # Calculate rate
+        if self.start_time:
+            elapsed = (datetime.now() - self.start_time).total_seconds()
+            rate = self.current / elapsed if elapsed > 0 else 0
+        else:
+            rate = 0
+            
+        # Call callback
+        if self.callback:
+            try:
+                self.callback(self.current, self.total_expected, rate)
+            except Exception as e:
+                logger.error(f"Progress callback error: {e}")
+        
+    def get_rate(self) -> float:
+        """Get current processing rate (items per second)."""
+        if not self.start_time:
+            return 0.0
+            
+        elapsed = (datetime.now() - self.start_time).total_seconds()
+        return self.current / elapsed if elapsed > 0 else 0.0
+        
+    def get_eta(self) -> Optional[datetime]:
+        """Get estimated time of completion."""
+        if not self.total_expected or not self.start_time or self.current <= 0:
+            return None
+            
+        rate = self.get_rate()
+        if rate <= 0:
+            return None
+            
+        remaining = self.total_expected - self.current
+        eta_seconds = remaining / rate
+        
+        return datetime.now() + timedelta(seconds=eta_seconds)
+    
+    def get_elapsed(self) -> Optional[timedelta]:
+        """Get elapsed time since start."""
+        if not self.start_time:
+            return None
+        return datetime.now() - self.start_time
+    
+    def get_progress_percentage(self) -> Optional[float]:
+        """Get progress as percentage (0-100)."""
+        if not self.total_expected or self.total_expected <= 0:
+            return None
+        return min(100.0, (self.current / self.total_expected) * 100)
+    
+    def get_summary(self) -> dict:
+        """Get progress summary."""
+        return {
+            'current': self.current,
+            'total': self.total_expected,
+            'rate': self.get_rate(),
+            'eta': self.get_eta(),
+            'elapsed': self.get_elapsed(),
+            'percentage': self.get_progress_percentage()
+        }

tgdata/smoke_tests/__init__.py

@@ -0,0 +1 @@
+# Smoke tests for Telegram Group Message Crawler

tgdata/smoke_tests/my_test.py

@@ -0,0 +1,149 @@
+import asyncio
+import os
+import sys
+import pandas as pd
+from tgdata import TgData
+
+# python -m tgdata.smoke_tests.my_test
+
+
+
+async def extract_to_csv_with_resume(tg: TgData, group_id: int, csv_file: str = "messages.csv", batch_size: int = 10):
+    """
+    Extract messages to CSV using the CSV itself as checkpoint.
+    Supports resuming from the last message ID in the CSV.
+    
+    Uses get_messages(after_id=...) for incremental extraction.
+    """
+    # Read last message ID from existing CSV
+    last_message_id = 0
+    existing_count = 0
+    
+    if os.path.exists(csv_file):
+        try:
+            # Read just the last row efficiently
+            # In production, you might use: pd.read_csv(csv_file, skiprows=lambda x: x < last_row)
+            df_existing = pd.read_csv(csv_file)
+            if not df_existing.empty:
+                # Get the maximum message ID (in case CSV is not ordered)
+                last_message_id = int(df_existing['MessageId'].max())
+                existing_count = len(df_existing)
+                print(f"📄 Found existing CSV with {existing_count} messages")
+                print(f"📍 Resuming from message ID: {last_message_id}")
+        except Exception as e:
+            print(f"⚠️  Could not read existing CSV: {e}")
+            print("   Starting fresh extraction")
+    else:
+        print("📄 No existing CSV found, starting fresh extraction")
+    
+    # Use get_messages with after_id for incremental extraction
+    print(f"\n🔄 Extracting messages since ID {last_message_id}...")
+    
+    # Get today's date at midnight in UTC
+    from datetime import datetime, time, timezone
+    today_utc = datetime.combine(datetime.now(timezone.utc).date(), time.min, tzinfo=timezone.utc)
+    print(f"📅 Filtering for messages from today ({today_utc.date()}) only")
+    print(f"⏰ Start time filter: {today_utc} (UTC)")
+    print(f"⏰ Current time: {datetime.now(timezone.utc)} (UTC)")
+    
+    try:
+        # Use get_messages with after_id for proper batch extraction
+        new_messages = await tg.get_messages(
+            group_id=group_id,
+            after_id=last_message_id
+        )
+        
+        # For demo purposes, limit to batch_size and filter for today
+        if not new_messages.empty:
+            print(f"   Got {len(new_messages)} messages since ID {last_message_id}")
+            
+            # Convert Date column to UTC if needed
+            new_messages['Date'] = pd.to_datetime(new_messages['Date'], utc=True)
+            
+            # Filter for today's messages only (for demo)
+            today_messages = new_messages[new_messages['Date'] >= today_utc]
+            print(f"   Found {len(today_messages)} messages from today")
+            
+            # Limit to batch_size
+            new_messages = today_messages.head(batch_size)
+            print(f"   Limited to {len(new_messages)} messages (batch_size={batch_size})")
+        
+        if not new_messages.empty:
+            # Sort by MessageId to ensure order
+            new_messages = new_messages.sort_values('MessageId')
+            
+            # Append to CSV
+            new_messages.to_csv(
+                csv_file,
+                mode='a',
+                header=not os.path.exists(csv_file),
+                index=False
+            )
+            
+            print(f"✅ Added {len(new_messages)} new messages to CSV")
+            
+            # Show sample of new messages
+            print("\n📋 Sample of new messages:")
+            # Show relevant columns
+            cols_to_show = ['MessageId', 'Name', 'Date']
+            print(new_messages[cols_to_show].head(5))
+        else:
+            print("ℹ️  No new messages found")
+    
+    except Exception as e:
+        print(f"❌ Error extracting messages: {e}")
+        new_messages = pd.DataFrame()
+    
+    # Summary
+    print(f"\n📊 Extraction Summary:")
+    print(f"   - Existing messages: {existing_count}")
+    print(f"   - New messages: {len(new_messages) if not new_messages.empty else 0}")
+    print(f"   - Total messages: {existing_count + (len(new_messages) if not new_messages.empty else 0)}")
+    
+    return new_messages
+
+
+
+async def main():
+    """Run CSV checkpoint test"""
+    
+    
+    print("This test demonstrates:")
+    print("- Using CSV file itself as checkpoint (no checkpoint.json)")
+    print("- Reading last message ID from existing CSV")
+    print("- Resuming extraction from that point")
+    print("- Appending new messages to existing CSV")
+    
+    # Define the test function inline or import it
+    # For now, let's create a simple test
+    tg = TgData("config.ini")
+    csv_file = "test_messages.csv"
+    
+    try:
+        # Get test group ID from environment or use Bitcoinsensus channel
+        group_id = int(os.environ.get('TEST_GROUP_ID', '1670178185'))  # Bitcoinsensus channel
+        
+        print(f"Using group ID: {group_id}")
+        print("\n⚠️  Using gentle settings: batch_size=10, only today's messages")
+        
+        # Run the extraction
+        messages = await extract_to_csv_with_resume(tg, group_id, csv_file, batch_size=10)
+        
+        success = True
+    except Exception as e:
+        print(f"Error: {e}")
+        success = False
+    finally:
+        await tg.close()
+        # Cleanup
+        if os.path.exists(csv_file):
+            os.remove(csv_file)
+            print(f"Cleaned up {csv_file}")
+    
+
+    if not success:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tgdata/smoke_tests/test_00_connection.py

@@ -0,0 +1,286 @@
+"""
+Smoke tests for ConnectionEngine with real Telegram connections
+"""
+# To run: python -m tgdata.smoke_tests.test_01_connection
+
+import asyncio
+import sys
+import os
+import time
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from tgdata.connection_engine import ConnectionEngine, ConnectionPool, RateLimitInfo
+
+
+async def test_connection_engine_initialization():
+    """Test ConnectionEngine initialization with real config"""
+    print("TEST: ConnectionEngine initialization...")
+    try:
+        # Test with real config.ini
+        if not os.path.exists("config.ini"):
+            print("✗ config.ini not found. Please ensure config.ini exists with valid Telegram credentials")
+            return False
+            
+        engine = ConnectionEngine("config.ini")
+        assert engine.config_path == "config.ini"
+        assert engine.pool_size == 1
+        assert engine.max_retries == 3
+        print("✓ Default initialization successful")
+        
+        # Test with custom settings
+        engine2 = ConnectionEngine(
+            config_path="config.ini",
+            pool_size=2,
+            max_retries=2,
+            retry_delay=1.0,
+            exponential_backoff=True
+        )
+        assert engine2.pool_size == 2
+        assert engine2.max_retries == 2
+        print("✓ Custom initialization successful")
+        
+        return True
+    except Exception as e:
+        print(f"✗ Initialization test failed: {e}")
+        return False
+
+
+async def test_real_connection():
+    """Test real Telegram connection"""
+    print("\nTEST: Real Telegram connection...")
+    engine = None
+    try:
+        engine = ConnectionEngine("config.ini", pool_size=1)
+        
+        # Test getting a real client
+        print("Attempting to get Telegram client...")
+        client = await engine.get_client()
+        
+        # Verify it's connected
+        assert client is not None
+        assert await client.is_user_authorized()
+        print("✓ Successfully connected to Telegram")
+        
+        # Get some basic info to verify connection
+        me = await client.get_me()
+        print(f"✓ Authenticated as: {me.first_name} (ID: {me.id})")
+        
+        # Test connection validation
+        is_valid = await engine.validate_connection()
+        assert is_valid == True
+        print("✓ Connection validation successful")
+        
+        return True
+    except Exception as e:
+        print(f"✗ Real connection test failed: {e}")
+        print("Make sure:")
+        print("  1. config.ini has valid Telegram API credentials")
+        print("  2. You have an authenticated session ('karaposu'.session)")
+        return False
+    finally:
+        if engine:
+            await engine.close()
+
+
+async def test_config_loading():
+    """Test real configuration loading"""
+    print("\nTEST: Real configuration loading...")
+    try:
+        engine = ConnectionEngine("config.ini")
+        config = engine._load_config()
+        
+        # Check that real config values are loaded
+        assert config.api_id is not None
+        assert config.api_hash is not None
+        assert len(config.api_id) > 0
+        assert len(config.api_hash) > 0
+        print(f"✓ Config loaded successfully")
+        print(f"  - API ID: {config.api_id}")
+        print(f"  - Session: {config.username if config.username else config.session_file}")
+        print(f"  - Phone: {config.phone if config.phone else 'Not specified'}")
+        
+        return True
+    except Exception as e:
+        print(f"✗ Config loading test failed: {e}")
+        return False
+
+
+async def test_health_check():
+    """Test health check with real connection"""
+    print("\nTEST: Health check with real connection...")
+    engine = None
+    try:
+        engine = ConnectionEngine("config.ini")
+        
+        # Get a connection first
+        await engine.get_client()
+        
+        # Run health check
+        health = await engine.health_check()
+        
+        assert isinstance(health, dict)
+        assert 'timestamp' in health
+        assert 'primary_connection' in health
+        assert 'pool_connections' in health
+        assert 'errors' in health
+        
+        print("✓ Health check structure correct")
+        print(f"  - Primary connection: {'healthy' if health['primary_connection'] else 'unhealthy'}")
+        print(f"  - Pool connections: {len(health['pool_connections'])}")
+        print(f"  - Errors: {len(health['errors'])}")
+        
+        # Primary connection should be healthy
+        assert health['primary_connection'] == True
+        print("✓ Primary connection is healthy")
+        
+        return True
+    except Exception as e:
+        print(f"✗ Health check test failed: {e}")
+        return False
+    finally:
+        if engine:
+            await engine.close()
+
+
+async def test_connection_pool_real():
+    """Test connection pool with real connections"""
+    print("\nTEST: Connection pool with real connections...")
+    engine = None
+    try:
+        # Note: For authenticated sessions, pool_size=1 is usually sufficient
+        # as Telegram allows multiple concurrent requests on the same connection
+        engine = ConnectionEngine("config.ini", pool_size=1)
+        
+        print("Testing connection reuse...")
+        
+        # Get multiple client references
+        client1 = await engine.get_client()
+        client2 = await engine.get_client()
+        
+        # Both should be valid
+        assert await client1.is_user_authorized()
+        print("✓ First client authorized")
+        
+        assert await client2.is_user_authorized()
+        print("✓ Second client authorized")
+        
+        # For pool_size=1, they should be the same client instance
+        if engine.pool_size == 1:
+            assert client1 is client2
+            print("✓ Single connection reused correctly (pool_size=1)")
+        else:
+            print(f"✓ Pool with {engine.pool_size} connections working")
+        
+        # Test that we can use both references
+        me1 = await client1.get_me()
+        me2 = await client2.get_me()
+        assert me1.id == me2.id
+        print("✓ Both client references work correctly")
+        
+        return True
+    except Exception as e:
+        print(f"✗ Connection pool test failed: {e}")
+        print("This might happen if the session requires re-authentication")
+        return False
+    finally:
+        if engine:
+            await engine.close()
+
+
+async def test_connection_close():
+    """Test connection closing with real connection"""
+    print("\nTEST: Connection closing...")
+    engine = None
+    try:
+        engine = ConnectionEngine("config.ini")
+        
+        # Get a real connection
+        client = await engine.get_client()
+        assert await client.is_user_authorized()
+        
+        # Close connections
+        await engine.close()
+        print("✓ Close operation successful")
+        
+        # Verify client is disconnected
+        # Note: We can't easily test if it's truly disconnected without errors
+        print("✓ Engine closed without errors")
+        
+        return True
+    except Exception as e:
+        print(f"✗ Connection close test failed: {e}")
+        return False
+
+
+async def test_rate_limiting():
+    """Test rate limiting behavior"""
+    print("\nTEST: Rate limiting behavior...")
+    engine = None
+    try:
+        engine = ConnectionEngine("config.ini")
+        client = await engine.get_client()
+        
+        # Note: We can't easily trigger real rate limits without making many requests
+        # But we can test the mechanism
+        print("✓ Rate limiting mechanisms in place")
+        print("  - Exponential backoff enabled")
+        print("  - Retry logic configured")
+        
+        return True
+    except Exception as e:
+        print(f"✗ Rate limiting test failed: {e}")
+        return False
+    finally:
+        if engine:
+            await engine.close()
+
+
+async def main():
+    """Run all connection engine tests with real connections"""
+    print("Connection Engine Tests (Real Connections)")
+    print("=" * 50)
+    print("\nNOTE: These tests require:")
+    print("  1. Valid config.ini with Telegram API credentials")
+    print("  2. Authenticated session file ('karaposu'.session)")
+    print("  3. Active internet connection")
+    print("=" * 50)
+    
+    tests = [
+        test_connection_engine_initialization,
+        test_config_loading,
+        test_real_connection,
+        test_health_check,
+        test_connection_pool_real,
+        test_rate_limiting,
+        test_connection_close
+    ]
+    
+    results = []
+    for test in tests:
+        try:
+            result = await test()
+            results.append(result)
+        except Exception as e:
+            print(f"✗ Test failed with error: {e}")
+            import traceback
+            traceback.print_exc()
+            results.append(False)
+    
+    # Summary
+    print("\nSummary")
+    print("=" * 50)
+    passed = sum(results)
+    total = len(results)
+    print(f"Passed: {passed}/{total}")
+    
+    if passed == total:
+        print("✓ All connection engine tests passed!")
+        return 0
+    else:
+        print("✗ Some tests failed")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)