pytrends-modern 0.2.0__tar.gz → 0.2.2__tar.gz

{pytrends_modern-0.2.0/pytrends_modern.egg-info → pytrends_modern-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrends-modern
-Version: 0.2.0
+Version: 0.2.2
 Summary: Modern Google Trends API - Combining the best of pytrends, with RSS feeds, Selenium scraping, DrissionPage browser automation, and enhanced features
 Author: pytrends-modern contributors
 License: MIT
@@ -175,11 +175,16 @@ RUN mkdir -p /root/.config && \
     cd /root/.config && \
     tar -xzf /tmp/profile.tar.gz
 
-# 3. Use headless mode in container
-config = BrowserConfig(headless=True)
+# 3. Use headless="virtual" in container
+config = BrowserConfig(headless="virtual")  # Use Xvfb for Docker
 ```
 
-See `Dockerfile.example` for complete Docker setup.
+**Headless Options:**
+- `headless=False` - Show browser window (local development)
+- `headless=True` - Standard headless (servers with display)
+- `headless="virtual"` - Xvfb virtual display (Docker containers)
+
+See `Dockerfile.example` and `examples/example_docker_usage.py` for complete Docker setup.
 
 ⚠️ **Security**: Profile contains Google session - keep secure, don't commit to git!
 

{pytrends_modern-0.2.0 → pytrends_modern-0.2.2}/README.md

@@ -125,11 +125,16 @@ RUN mkdir -p /root/.config && \
     cd /root/.config && \
     tar -xzf /tmp/profile.tar.gz
 
-# 3. Use headless mode in container
-config = BrowserConfig(headless=True)
+# 3. Use headless="virtual" in container
+config = BrowserConfig(headless="virtual")  # Use Xvfb for Docker
 ```
 
-See `Dockerfile.example` for complete Docker setup.
+**Headless Options:**
+- `headless=False` - Show browser window (local development)
+- `headless=True` - Standard headless (servers with display)
+- `headless="virtual"` - Xvfb virtual display (Docker containers)
+
+See `Dockerfile.example` and `examples/example_docker_usage.py` for complete Docker setup.
 
 ⚠️ **Security**: Profile contains Google session - keep secure, don't commit to git!
 

pytrends_modern-0.2.2/examples/example_docker_usage.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""
+Example: Using pytrends-modern in Docker containers
+
+When running in Docker, use headless="virtual" to enable Xvfb virtual display.
+This prevents display errors in containerized environments.
+"""
+
+import asyncio
+from pytrends_modern import AsyncTrendReq, BrowserConfig
+
+
+async def main():
+    """Example async usage in Docker container"""
+    
+    # Configure for Docker environment
+    config = BrowserConfig(
+        headless="virtual",  # Use Xvfb virtual display for Docker
+        humanize=True,
+        os='linux',
+        geoip=True
+    )
+    
+    # For local development, use:
+    # config = BrowserConfig(headless=False)  # Show browser window
+    # config = BrowserConfig(headless=True)   # Standard headless
+    
+    print("🐳 Running pytrends-modern in Docker container...")
+    print(f"📁 Profile: {config.user_data_dir}")
+    
+    async with AsyncTrendReq(browser_config=config) as pytrends:
+        pytrends.kw_list = ["Docker"]
+        
+        # Get interest over time
+        print("\n📊 Fetching interest_over_time...")
+        df = await pytrends.interest_over_time()
+        print(f"✓ Got {len(df)} rows")
+        print(df.head())
+        
+        # Get interest by region
+        print("\n🌍 Fetching interest_by_region...")
+        df_region = await pytrends.interest_by_region()
+        print(f"✓ Got {len(df_region)} rows")
+        print(df_region.head())
+    
+    print("\n✅ Complete!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

pytrends_modern-0.2.2/examples/test_async_integration.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""
+Test AsyncTrendReq with Camoufox
+"""
+
+import asyncio
+from pytrends_modern import AsyncTrendReq, BrowserConfig
+
+async def test_async_browser_mode():
+    """Test async browser mode with all 4 APIs"""
+    print("=" * 70)
+    print("Testing AsyncTrendReq with Camoufox")
+    print("=" * 70)
+    
+    # Configure browser
+    config = BrowserConfig(
+        headless=False,
+        humanize=True,
+        os='linux',
+        geoip=True
+    )
+    
+    keyword = "Python"
+    print(f"\n🔍 Testing with keyword: {keyword}")
+    
+    # Use async context manager
+    async with AsyncTrendReq(browser_config=config) as pytrends:
+        pytrends.kw_list = [keyword]
+        
+        # Test interest_over_time
+        print("\n📊 Testing interest_over_time()...")
+        try:
+            df_iot = await pytrends.interest_over_time()
+            print(f"✓ interest_over_time: {len(df_iot)} rows")
+            print(df_iot.head())
+        except Exception as e:
+            print(f"✗ Error: {e}")
+        
+        # Test interest_by_region
+        print("\n🌍 Testing interest_by_region()...")
+        try:
+            df_ibr = await pytrends.interest_by_region()
+            print(f"✓ interest_by_region: {len(df_ibr)} rows")
+            print(df_ibr.head())
+        except Exception as e:
+            print(f"✗ Error: {e}")
+        
+        # Test related_topics
+        print("\n🔗 Testing related_topics()...")
+        try:
+            topics = await pytrends.related_topics()
+            if keyword in topics and topics[keyword].get('top') is not None:
+                print(f"✓ related_topics: {len(topics[keyword]['top'])} topics")
+                print(topics[keyword]['top'].head())
+            else:
+                print("✓ related_topics: No data (might be normal)")
+        except Exception as e:
+            print(f"✗ Error: {e}")
+        
+        # Test related_queries
+        print("\n❓ Testing related_queries()...")
+        try:
+            queries = await pytrends.related_queries()
+            if keyword in queries and queries[keyword].get('top') is not None:
+                print(f"✓ related_queries: {len(queries[keyword]['top'])} queries")
+                print(queries[keyword]['top'].head())
+            else:
+                print("✓ related_queries: No data (might be normal)")
+        except Exception as e:
+            print(f"✗ Error: {e}")
+    
+    print("\n" + "=" * 70)
+    print("✅ Async test complete!")
+    print("=" * 70)
+
+if __name__ == "__main__":
+    # Run async test
+    asyncio.run(test_async_browser_mode())

{pytrends_modern-0.2.0 → pytrends_modern-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pytrends-modern"
-version = "0.2.0"
+version = "0.2.2"
 description = "Modern Google Trends API - Combining the best of pytrends, with RSS feeds, Selenium scraping, DrissionPage browser automation, and enhanced features"
 readme = "README.md"
 requires-python = ">=3.8"

{pytrends_modern-0.2.0 → pytrends_modern-0.2.2}/pytrends_modern/__init__.py

@@ -2,11 +2,12 @@
 pytrends-modern: Modern Google Trends API
 """
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"
 __author__ = "pytrends-modern contributors"
 __license__ = "MIT"
 
 from pytrends_modern.request import TrendReq
+from pytrends_modern.request_async import AsyncTrendReq
 from pytrends_modern.rss import TrendsRSS
 from pytrends_modern.scraper import TrendsScraper
 from pytrends_modern.browser_config_camoufox import BrowserConfig
@@ -21,6 +22,7 @@ from pytrends_modern.exceptions import (
 
 __all__ = [
     "TrendReq",
+    "AsyncTrendReq",
     "TrendsRSS",
     "TrendsScraper",
     "BrowserConfig",

{pytrends_modern-0.2.0 → pytrends_modern-0.2.2}/pytrends_modern/browser_config_camoufox.py

@@ -1,6 +1,6 @@
 """Browser configuration for Camoufox automation"""
 
-from typing import Optional
+from typing import Optional, Union
 import os as os_module
 
 
@@ -18,7 +18,9 @@ class BrowserConfig:
     
     Args:
         headless: Run browser in headless mode (default: False)
-                 Set to 'virtual' on Linux to use Xvfb
+                 - False: Show browser window (for local development)
+                 - True: Standard headless mode (for servers with display)
+                 - 'virtual': Use Xvfb virtual display (for Docker containers)
         proxy_server: Proxy server URL (e.g., 'http://proxy.com:8080')
         proxy_username: Proxy username (for authenticated proxies)
         proxy_password: Proxy password (for authenticated proxies)
@@ -47,7 +49,7 @@ class BrowserConfig:
     
     def __init__(
         self,
-        headless: bool = False,
+        headless: Union[bool, str] = False,
         proxy_server: Optional[str] = None,
         proxy_username: Optional[str] = None,
         proxy_password: Optional[str] = None,

pytrends_modern-0.2.2/pytrends_modern/request_async.py

@@ -0,0 +1,342 @@
+"""
+Async Google Trends API request module with Camoufox support
+"""
+
+import json
+from typing import Dict, Optional
+from urllib.parse import quote
+
+import pandas as pd
+
+from pytrends_modern import exceptions
+from pytrends_modern.browser_config_camoufox import BrowserConfig
+
+
+class AsyncTrendReq:
+    """
+    Async Google Trends API with Camoufox browser mode support
+    
+    This class provides async methods for fetching Google Trends data
+    using Camoufox's async API for better performance in async applications.
+    
+    Example:
+        >>> import asyncio
+        >>> from pytrends_modern import AsyncTrendReq, BrowserConfig
+        >>> 
+        >>> async def main():
+        ...     config = BrowserConfig(headless=True)
+        ...     async with AsyncTrendReq(browser_config=config) as pytrends:
+        ...         pytrends.kw_list = ['Python']
+        ...         df = await pytrends.interest_over_time()
+        ...         print(df.head())
+        >>> 
+        >>> asyncio.run(main())
+    """
+    
+    def __init__(self, browser_config: BrowserConfig):
+        """
+        Initialize async Google Trends request
+        
+        Args:
+            browser_config: BrowserConfig instance for Camoufox
+        """
+        self.browser_config = browser_config
+        self.browser = None
+        self.browser_context = None
+        self.browser_page = None
+        self.browser_responses_cache = {}
+        self.kw_list = []
+        
+    async def __aenter__(self):
+        """Async context manager entry"""
+        await self._init_camoufox()
+        return self
+        
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit"""
+        await self._close_browser()
+        
+    async def _init_camoufox(self) -> None:
+        """Initialize Camoufox browser with persistent context (async)"""
+        try:
+            from camoufox.async_api import AsyncCamoufox
+        except ImportError:
+            raise ImportError(
+                "Camoufox is required for async browser mode. "
+                "Install with: pip install pytrends-modern[browser]"
+            )
+        
+        # Prepare browser options
+        import os
+        user_data_dir = os.path.expanduser(
+            self.browser_config.user_data_dir or "~/.config/camoufox-pytrends-profile"
+        )
+        
+        # Check if profile is configured (has Google login)
+        from pytrends_modern.camoufox_setup import is_profile_configured
+        if not is_profile_configured(user_data_dir):
+            raise exceptions.BrowserError(
+                f"Camoufox profile not configured at: {user_data_dir}\n"
+                "You must set up your Google account login first:\n\n"
+                "  from pytrends_modern.camoufox_setup import setup_profile\n"
+                "  setup_profile()\n\n"
+                "Or run from command line:\n"
+                "  python -m pytrends_modern.camoufox_setup\n\n"
+                "This will open a browser for you to log in to Google."
+            )
+        
+        # Proxy configuration (if provided)
+        proxy_config = None
+        if self.browser_config.proxy_server:
+            proxy_config = {
+                "server": self.browser_config.proxy_server,
+            }
+            if self.browser_config.proxy_username:
+                proxy_config["username"] = self.browser_config.proxy_username
+            if self.browser_config.proxy_password:
+                proxy_config["password"] = self.browser_config.proxy_password
+        
+        # Initialize AsyncCamoufox with persistent context
+        try:
+            # AsyncCamoufox() returns a context manager
+            camoufox_manager = AsyncCamoufox(
+                persistent_context=True,
+                user_data_dir=user_data_dir,
+                headless=self.browser_config.headless,
+                humanize=self.browser_config.humanize if hasattr(self.browser_config, 'humanize') else True,
+                os=self.browser_config.os if hasattr(self.browser_config, 'os') else 'linux',
+                geoip=self.browser_config.geoip if hasattr(self.browser_config, 'geoip') else True,
+                proxy=proxy_config
+            )
+            
+            # Enter the context manager to get the browser context
+            self.browser = camoufox_manager
+            self.browser_context = await camoufox_manager.__aenter__()
+            
+            # Use existing page if available (avoid opening 2 tabs)
+            if self.browser_context.pages:
+                self.browser_page = self.browser_context.pages[0]
+            else:
+                self.browser_page = await self.browser_context.new_page()
+            
+            # Set up network interception
+            self.browser_page.on("response", self._handle_network_response)
+            
+        except Exception as e:
+            raise exceptions.BrowserError(f"Failed to initialize AsyncCamoufox: {e}")
+    
+    async def _close_browser(self) -> None:
+        """Close browser if open (async)"""
+        if self.browser:
+            try:
+                # Exit the context manager
+                await self.browser.__aexit__(None, None, None)
+            except Exception:
+                pass
+            self.browser = None
+            self.browser_context = None
+            self.browser_page = None
+    
+    async def _handle_network_response(self, response) -> None:
+        """
+        Handle network responses and cache Google Trends API data (async)
+        
+        Args:
+            response: Playwright response object
+        """
+        url = response.url
+        
+        # Only process Google Trends API responses
+        if '/trends/api/widgetdata/' not in url:
+            return
+        
+        try:
+            # Get response body (async in AsyncPlaywright)
+            body = await response.body()
+            
+            # Parse the response (remove Google's JSONP prefix - exactly 5 bytes)
+            if body.startswith(b")]}'\n"):
+                body = body[5:]
+            elif body.startswith(b")]}'"):
+                body = body[5:]
+            
+            data = json.loads(body)
+            
+            # Cache by URL pattern
+            if '/widgetdata/multiline' in url:
+                self.browser_responses_cache['interest_over_time'] = data
+            elif '/widgetdata/comparedgeo' in url:
+                self.browser_responses_cache['interest_by_region'] = data
+            elif '/widgetdata/relatedsearches' in url:
+                # keywordType is URL-encoded inside the req parameter
+                import urllib.parse
+                decoded_url = urllib.parse.unquote(url)
+                if 'keywordType":"ENTITY' in decoded_url:
+                    self.browser_responses_cache['related_topics'] = data
+                elif 'keywordType":"QUERY' in decoded_url:
+                    self.browser_responses_cache['related_queries'] = data
+                    
+        except Exception:
+            pass  # Silently ignore parsing errors
+    
+    async def _capture_all_api_responses(self, keyword: str) -> None:
+        """
+        Navigate once and capture ALL API responses via network interception (async)
+        
+        Args:
+            keyword: Search keyword to use
+        """
+        if not self.browser_page:
+            raise exceptions.BrowserError("Browser not initialized")
+        
+        # Clear cache
+        self.browser_responses_cache.clear()
+        
+        # Build URL
+        import urllib.parse
+        encoded_keyword = urllib.parse.quote(keyword)
+        url = f"https://trends.google.com/trends/explore?date=today%201-m&q={encoded_keyword}&hl=en-GB"
+        
+        try:
+            # Navigate and wait for network idle
+            await self.browser_page.goto(url, wait_until='networkidle', timeout=60000)
+            
+            # Give extra time for any delayed API calls
+            import asyncio
+            await asyncio.sleep(2)
+            
+        except Exception as e:
+            raise exceptions.BrowserError(f"Failed to navigate to Google Trends: {e}")
+    
+    def _parse_multiline_response(self, data: Dict) -> pd.DataFrame:
+        """Parse multiline (interest over time) API response"""
+        # Import from sync version to reuse parsing logic
+        from pytrends_modern.request import TrendReq
+        temp = TrendReq(hl='en-US', tz=360)
+        return temp._parse_multiline_response(data)
+    
+    def _parse_comparedgeo_response(self, data: Dict, inc_geo_code: bool = False) -> pd.DataFrame:
+        """Parse comparedgeo (interest by region) API response"""
+        from pytrends_modern.request import TrendReq
+        temp = TrendReq(hl='en-US', tz=360)
+        return temp._parse_comparedgeo_response(data, inc_geo_code)
+    
+    def _parse_relatedsearches_response(self, data: Dict) -> Dict[str, Optional[pd.DataFrame]]:
+        """Parse relatedsearches (related topics/queries) API response"""
+        from pytrends_modern.request import TrendReq
+        temp = TrendReq(hl='en-US', tz=360)
+        return temp._parse_relatedsearches_response(data)
+    
+    async def interest_over_time(self) -> pd.DataFrame:
+        """
+        Get interest over time data (async)
+        
+        Returns:
+            DataFrame with date index and columns for each keyword
+        """
+        if len(self.kw_list) != 1:
+            raise exceptions.InvalidParameterError(
+                "Async browser mode only supports 1 keyword. You provided: "
+                + str(len(self.kw_list))
+            )
+        
+        keyword = self.kw_list[0]
+        
+        # Capture all responses if not already cached
+        if not self.browser_responses_cache:
+            await self._capture_all_api_responses(keyword)
+        
+        # Get cached response
+        response_data = self.browser_responses_cache.get('interest_over_time')
+        
+        if not response_data:
+            # Try one more navigation if cache is empty
+            await self._capture_all_api_responses(keyword)
+            response_data = self.browser_responses_cache.get('interest_over_time')
+            
+        if not response_data:
+            raise exceptions.ResponseError("Failed to capture interest_over_time API response")
+        
+        # Parse browser response to DataFrame
+        return self._parse_multiline_response(response_data)
+    
+    async def interest_by_region(self, inc_geo_code: bool = False) -> pd.DataFrame:
+        """
+        Get interest by region data (async)
+        
+        Args:
+            inc_geo_code: Include geographic codes in results
+            
+        Returns:
+            DataFrame with region index
+        """
+        if len(self.kw_list) != 1:
+            raise exceptions.InvalidParameterError(
+                "Async browser mode only supports 1 keyword"
+            )
+        
+        keyword = self.kw_list[0]
+        
+        # Capture all responses if not already cached
+        if not self.browser_responses_cache:
+            await self._capture_all_api_responses(keyword)
+        
+        # Get cached response
+        response_data = self.browser_responses_cache.get('interest_by_region')
+        
+        if not response_data:
+            raise exceptions.ResponseError("Failed to capture interest_by_region API response")
+        
+        return self._parse_comparedgeo_response(response_data, inc_geo_code)
+    
+    async def related_topics(self) -> Dict[str, Dict[str, Optional[pd.DataFrame]]]:
+        """
+        Get related topics (async)
+        
+        Returns:
+            Dict with keyword as key and dict of 'top'/'rising' DataFrames as value
+        """
+        if len(self.kw_list) != 1:
+            raise exceptions.InvalidParameterError(
+                "Async browser mode only supports 1 keyword"
+            )
+        
+        keyword = self.kw_list[0]
+        
+        # Capture all responses if not already cached
+        if not self.browser_responses_cache:
+            await self._capture_all_api_responses(keyword)
+        
+        # Get cached response
+        response_data = self.browser_responses_cache.get('related_topics')
+        
+        if not response_data:
+            raise exceptions.ResponseError("Failed to capture related_topics API response")
+        
+        return {keyword: self._parse_relatedsearches_response(response_data)}
+    
+    async def related_queries(self) -> Dict[str, Dict[str, Optional[pd.DataFrame]]]:
+        """
+        Get related queries (async)
+        
+        Returns:
+            Dict with keyword as key and dict of 'top'/'rising' DataFrames as value
+        """
+        if len(self.kw_list) != 1:
+            raise exceptions.InvalidParameterError(
+                "Async browser mode only supports 1 keyword"
+            )
+        
+        keyword = self.kw_list[0]
+        
+        # Capture all responses if not already cached
+        if not self.browser_responses_cache:
+            await self._capture_all_api_responses(keyword)
+        
+        # Get cached response
+        response_data = self.browser_responses_cache.get('related_queries')
+        
+        if not response_data:
+            raise exceptions.ResponseError("Failed to capture related_queries API response")
+        
+        return {keyword: self._parse_relatedsearches_response(response_data)}

{pytrends_modern-0.2.0 → pytrends_modern-0.2.2/pytrends_modern.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pytrends-modern
-Version: 0.2.0
+Version: 0.2.2
 Summary: Modern Google Trends API - Combining the best of pytrends, with RSS feeds, Selenium scraping, DrissionPage browser automation, and enhanced features
 Author: pytrends-modern contributors
 License: MIT
@@ -175,11 +175,16 @@ RUN mkdir -p /root/.config && \
     cd /root/.config && \
     tar -xzf /tmp/profile.tar.gz
 
-# 3. Use headless mode in container
-config = BrowserConfig(headless=True)
+# 3. Use headless="virtual" in container
+config = BrowserConfig(headless="virtual")  # Use Xvfb for Docker
 ```
 
-See `Dockerfile.example` for complete Docker setup.
+**Headless Options:**
+- `headless=False` - Show browser window (local development)
+- `headless=True` - Standard headless (servers with display)
+- `headless="virtual"` - Xvfb virtual display (Docker containers)
+
+See `Dockerfile.example` and `examples/example_docker_usage.py` for complete Docker setup.
 
 ⚠️ **Security**: Profile contains Google session - keep secure, don't commit to git!
 

{pytrends_modern-0.2.0 → pytrends_modern-0.2.2}/pytrends_modern.egg-info/SOURCES.txt

@@ -5,6 +5,8 @@ pyproject.toml
 examples/advanced_usage.py
 examples/basic_usage.py
 examples/example_browser_mode.py
+examples/example_docker_usage.py
+examples/test_async_integration.py
 pytrends_modern/__init__.py
 pytrends_modern/browser_config.py
 pytrends_modern/browser_config_camoufox.py
@@ -15,6 +17,7 @@ pytrends_modern/exceptions.py
 pytrends_modern/proxy_extension.py
 pytrends_modern/py.typed
 pytrends_modern/request.py
+pytrends_modern/request_async.py
 pytrends_modern/rss.py
 pytrends_modern/scraper.py
 pytrends_modern/utils.py