amzpy 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

amzpy/scraper.py

@@ -1,52 +1,223 @@
 """
-The main scraper module for the Amazon Product API.
+Amazon Product Scraper Module
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is the main module for the Amazon Product API using curl_cffi.
+It orchestrates the scraping workflow including:
+- Managing sessions through AmzSession
+- Fetching product details
+- Searching for products
+- Handling configuration
+
+The AmazonScraper class provides a simple interface for users
+while handling the complexity of Amazon's anti-bot measures underneath.
 """
 
-from typing import Dict, Optional
-from amzpy.engine import RequestEngine
-from amzpy.parser import parse_product_page
+from typing import Dict, Optional, List, Union, Any
+import re
+
+from amzpy.session import AmzSession, DEFAULT_CONFIG
+from amzpy.parser import parse_product_page, parse_search_page, parse_pagination_url
 from amzpy.utils import parse_amazon_url
 
+
 class AmazonScraper:
-    """Main scraper class for Amazon product data"""
+    """
+    Main scraper class for Amazon product data using curl_cffi.
+    
+    This class provides a high-level interface to:
+    - Fetch detailed information for individual products
+    - Search for products and extract listings
+    - Configure scraping behavior
     
-    def __init__(self, country_code: str = "com"):
-        """Initialize the Amazon scraper"""
-        self.base_url = f"https://www.amazon.{country_code}/"
-        self.engine = RequestEngine()
+    It handles browser impersonation, CAPTCHA avoidance, and parsing
+    through the session and parser modules.
     
-    def get_product_details(self, url: str, max_retries: int = 3) -> Optional[Dict]:
+    Attributes:
+        country_code (str): Amazon domain country code (e.g. "com", "in")
+        session (AmzSession): Session manager for handling requests
+        user_config (dict): User configuration parameters
+    """
+
+    def __init__(self, country_code: str = "com", impersonate: str = None, proxies: Optional[Dict] = None):
+        """
+        Initialize the Amazon scraper with the specified configuration.
+        
+        Args:
+            country_code (str): Amazon domain country code (e.g. "com", "in")
+            impersonate (str, optional): Browser to impersonate (e.g. "chrome119")
+            proxies (Dict, optional): Proxy configuration for requests
+        """
+        self.country_code = country_code
+        self.user_config = DEFAULT_CONFIG.copy()
+        self.session = AmzSession(
+            country_code=country_code,
+            impersonate=impersonate,
+            proxies=proxies
+        )
+        
+        print(f"AmazonScraper initialized for amazon.{country_code}")
+        
+    def config(self, config_str: str = None, **kwargs) -> Dict:
         """
-        Fetch details for a product using its Amazon URL
+        Configure scraper parameters using either a string or keyword arguments.
+        
+        Examples:
+            # Using string configuration
+            scraper.config('MAX_RETRIES = 5, REQUEST_TIMEOUT = 30')
+            
+            # Using keyword arguments
+            scraper.config(MAX_RETRIES=5, REQUEST_TIMEOUT=30)
         
         Args:
-            url (str): Amazon product URL
-            max_retries (int): Maximum retry attempts if anti-bot measures detected
+            config_str (str, optional): Configuration string in format 'PARAM1 = value1, PARAM2 = value2'
+            **kwargs: Configuration parameters as keyword arguments
             
         Returns:
-            Dict: Product details including title, price, img_url, and currency
-            None: If URL is invalid or error occurs
+            Dict: Current configuration after updates
         """
-        parsed = parse_amazon_url(url)
-        if not parsed:
-            return None
+        # Process string configuration if provided
+        if config_str:
+            # Parse the configuration string
+            try:
+                parts = config_str.split(',')
+                for part in parts:
+                    key, value = part.split('=', 1)
+                    key = key.strip()
+                    value = eval(value.strip())  # Safely evaluate the value
+                    self.user_config[key] = value
+            except Exception as e:
+                print(f"Error parsing configuration string: {e}")
+                print("Format should be: 'PARAM1 = value1, PARAM2 = value2'")
+        
+        # Process keyword arguments if provided
+        if kwargs:
+            self.user_config.update(kwargs)
+        
+        # Update the session configuration
+        self.session.update_config(**self.user_config)
+        
+        return self.user_config
+
+    def get_product_details(self, url: str) -> Optional[Dict]:
+        """
+        Fetch and parse details for a product using its Amazon URL.
+        
+        This method:
+        1. Parses the product URL to extract the ASIN
+        2. Constructs a canonical product URL
+        3. Fetches the product page HTML
+        4. Parses the HTML to extract structured data
+        
+        Args:
+            url (str): Amazon product URL (any format with a valid ASIN)
             
-        base_url, product_id = parsed
+        Returns:
+            Dict: Extracted product details (title, price, etc.)
+            None: If URL is invalid or scraping fails
+        """
+        # Parse the URL to extract base_url and product_id (ASIN)
+        parsed_info = parse_amazon_url(url)
+        if not parsed_info:
+            print(f"Invalid Amazon product URL: {url}")
+            return None
+
+        base_url, product_id = parsed_info
+        product_url = f"{base_url}dp/{product_id}"  # Construct canonical URL
+        print(f"Fetching product data for ASIN: {product_id}")
+
+        # Fetch the product page using the session
+        response = self.session.get(product_url)
+        if not response or not response.text:
+            print(f"Failed to fetch product page for: {product_url}")
+            return None
+        
+        # Parse the product page HTML, passing country code for URL formatting
+        product_data = parse_product_page(
+            html_content=response.text,
+            url=product_url,
+            country_code=self.country_code
+        )
         
-        # Construct product URL and get HTML
-        product_url = f"{base_url}dp/{product_id}"
-        html_content = self.engine.get(product_url)
-        if not html_content:
+        if not product_data:
+            print(f"Failed to extract product data from: {product_url}")
             return None
             
-        # Parse the product page and return the data
-        return parse_product_page(html_content, product_url, self.engine, max_retries)
-    
-def main():
-    scraper = AmazonScraper()
-    url = "https://www.amazon.in/dp/B0D4J2QDVY"
-    details = scraper.get_product_details(url, max_retries=5)
-    print("Product details:", details)
-
-if __name__ == "__main__":
-    main()
+        print(f"Successfully extracted data for: {product_data.get('title', 'Unknown Product')[:50]}...")
+        return product_data
+
+    def search_products(self, query: str = None, search_url: str = None, max_pages: int = 1) -> List[Dict]:
+        """
+        Search for products on Amazon and extract product listings.
+        
+        This method supports two search approaches:
+        1. Using a search query (e.g., "wireless headphones")
+        2. Using a pre-constructed search URL (e.g., category pages, filtered searches)
+        
+        It will automatically paginate through results up to max_pages.
+        
+        Args:
+            query (str, optional): Search query text (ignored if search_url is provided)
+            search_url (str, optional): Pre-constructed search URL (takes precedence over query)
+            max_pages (int): Maximum number of pages to scrape (default: 1)
+            
+        Returns:
+            List[Dict]: List of product data dictionaries from search results
+            Empty list: If search fails or no products are found
+        """
+        # Validate that we have either a query or a search URL
+        if not query and not search_url:
+            print("Error: Either a search query or search URL must be provided")
+            return []
+            
+        # Construct search URL if only query was provided
+        if not search_url and query:
+            search_url = f"https://www.amazon.{self.country_code}/s?k={query.replace(' ', '+')}"
+            
+        print(f"Starting product search: {search_url}")
+        
+        all_products = []  # Collect products from all pages
+        current_url = search_url
+        current_page = 1
+        
+        # Paginate through search results
+        while current_url and current_page <= max_pages:
+            print(f"\nScraping search page {current_page}/{max_pages}: {current_url}")
+            
+            # Fetch the search page
+            response = self.session.get(current_url)
+            if not response or not response.text:
+                print(f"Failed to fetch search page: {current_url}")
+                break
+                
+            # Parse products from the current page, passing country code for URL formatting
+            base_url = f"https://www.amazon.{self.country_code}"
+            products = parse_search_page(
+                response.text, 
+                base_url,
+                country_code=self.country_code
+            )
+            
+            # Check if we got valid results
+            if not products:
+                print(f"No products found on page {current_page} (or page was blocked)")
+                break
+                
+            print(f"Found {len(products)} products on page {current_page}")
+            all_products.extend(products)
+            
+            # Stop if we've reached the requested number of pages
+            if current_page >= max_pages:
+                break
+                
+            # Get URL for the next page
+            next_url = parse_pagination_url(response.text, base_url)
+            if not next_url:
+                print("No next page found. End of results.")
+                break
+                
+            current_url = next_url
+            current_page += 1
+            
+        print(f"\nSearch completed. Total products found: {len(all_products)}")
+        return all_products

amzpy/session.py

@@ -0,0 +1,222 @@
+"""
+Amazon Session Manager Module
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This module provides a robust session management system for Amazon scraping.
+It handles:
+- Browser impersonation with curl_cffi
+- Request retries with intelligent backoff
+- CAPTCHA/block detection and avoidance
+- User-agent rotation with fake_useragent
+- Proxy support
+"""
+
+import random
+import time
+from typing import Dict, Optional, Tuple, Any, Union
+
+import curl_cffi.requests
+from curl_cffi.requests.errors import RequestsError
+from fake_useragent import UserAgent
+
+# Default configuration (can be overridden by user)
+DEFAULT_CONFIG = {
+    'MAX_RETRIES': 3,
+    'REQUEST_TIMEOUT': 25,
+    'DELAY_BETWEEN_REQUESTS': (2, 5),
+    'DEFAULT_IMPERSONATE': 'chrome119'  # part of curl_cffi's impersonation
+}
+
+# Default header template
+DEFAULT_HEADERS = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Ch-Ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
+    'Sec-Ch-Ua-Mobile': '?0',
+    'Sec-Ch-Ua-Platform': '"Windows"',
+    'Sec-Fetch-Dest': 'document',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': '?1',
+}
+
+
+class AmzSession:
+    """
+    Enhanced session manager using curl_cffi for Amazon requests.
+    
+    This class implements sophisticated request handling including:
+    - Browser fingerprint spoofing (via curl_cffi impersonation)
+    - Randomized user agents (via fake_useragent)
+    - CAPTCHA/anti-bot detection and avoidance
+    - Intelligent retry logic with exponential backoff
+    - Proxy support for IP rotation
+    
+    Attributes:
+        country_code (str): Amazon domain country code (e.g., "com", "in", "co.uk")
+        base_url (str): Constructed base URL for the Amazon domain
+        session (curl_cffi.requests.Session): The curl_cffi session instance
+        config (dict): Configuration parameters for request behavior
+        ua_generator (UserAgent): User agent generator for browser fingerprinting
+    """
+
+    def __init__(self, country_code: str = "com", 
+                 impersonate: str = None, 
+                 proxies: Optional[Dict] = None,
+                 config: Optional[Dict] = None):
+        """
+        Initialize the Amazon session manager.
+        
+        Args:
+            country_code (str): Amazon domain country code (e.g. "com", "in")
+            impersonate (str, optional): Browser to impersonate (e.g. "chrome119")
+            proxies (Dict, optional): Proxy configuration for requests
+            config (Dict, optional): Override default configuration parameters
+        """
+        # Initialize country and base URL
+        self.country_code = country_code
+        self.base_url = f"https://www.amazon.{self.country_code}/"
+        
+        # Set up configuration (with user overrides if provided)
+        self.config = DEFAULT_CONFIG.copy()
+        if config:
+            self.config.update(config)
+        
+        # Initialize fake_useragent with common browser and OS combinations
+        self.ua_generator = UserAgent(browsers=['Chrome', 'Edge', 'Safari'], 
+                                     os=['Windows', 'MacOS', 'Linux'])
+        
+        # Create curl_cffi session
+        self.session = curl_cffi.requests.Session()
+        
+        # Set up headers with randomized user agent
+        headers = DEFAULT_HEADERS.copy()
+        headers['User-Agent'] = self.ua_generator.random
+        self.session.headers = headers
+        
+        # Set browser impersonation if provided, otherwise use default
+        self.session.impersonate = impersonate or self.config['DEFAULT_IMPERSONATE']
+        
+        # Configure proxies if provided
+        if proxies:
+            self.session.proxies = proxies
+            
+        # Print session initialization info
+        print(f"AmzSession initialized for amazon.{country_code}")
+        print(f"Impersonating: {self.session.impersonate}")
+        print(f"User-Agent: {headers['User-Agent'][:50]}...")
+        if proxies:
+            print(f"Using proxies: {proxies}")
+
+    def get(self, url: str, headers: Optional[Dict] = None) -> Optional[curl_cffi.requests.Response]:
+        """
+        Perform a GET request using the curl_cffi session with smart retries.
+        
+        This method intelligently handles:
+        - URL normalization (relative -> absolute)
+        - Header merging
+        - Random delays between requests
+        - CAPTCHA and anti-bot detection
+        - Automatic retries with exponential backoff
+        - Error handling for network issues
+        
+        Args:
+            url (str): URL to fetch (absolute or relative to base_url)
+            headers (Dict, optional): Additional headers to merge with defaults
+            
+        Returns:
+            Optional[curl_cffi.requests.Response]: Response object or None if all retries failed
+        """
+        # Normalize URL (handle both absolute and relative URLs)
+        if not url.startswith("http"):
+            if url.startswith("/"):
+                url = f"{self.base_url.rstrip('/')}{url}"
+            else:
+                url = f"{self.base_url}{url}"
+
+        # Merge headers with fresh random user agent for each request
+        merged_headers = self.session.headers.copy()
+        merged_headers['User-Agent'] = self.ua_generator.random
+        if headers:
+            merged_headers.update(headers)
+
+        # Extract configuration for use in the retry loop
+        max_retries = self.config['MAX_RETRIES']
+        timeout = self.config['REQUEST_TIMEOUT']
+        delay_range = self.config['DELAY_BETWEEN_REQUESTS']
+
+        # Retry loop with exponential backoff
+        for attempt in range(max_retries + 1):
+            try:
+                # Calculate delay with some randomization (increases with each attempt)
+                delay_factor = 1 + (attempt * 0.5)  # Exponential backoff factor
+                min_delay, max_delay = delay_range
+                delay = random.uniform(min_delay * delay_factor, max_delay * delay_factor)
+                
+                # Log attempt information
+                print(f"Request attempt {attempt+1}/{max_retries+1}: GET {url} (delay: {delay:.2f}s)")
+                time.sleep(delay)
+
+                # Make the actual request using curl_cffi
+                response = self.session.get(
+                    url,
+                    headers=merged_headers,
+                    timeout=timeout,
+                    allow_redirects=True
+                )
+
+                # Handle HTTP error codes
+                if response.status_code != 200:
+                    print(f"Non-200 status code: {response.status_code}")
+                    
+                    # Handle server errors specifically (5xx)
+                    if 500 <= response.status_code < 600 and attempt < max_retries:
+                        print(f"Server error {response.status_code}, retrying...")
+                        continue
+                    
+                    # For other status codes, continue but warn
+                    print(f"Warning: Received HTTP {response.status_code} for {url}")
+
+                # Check for CAPTCHA/blocking patterns in the content
+                if "captcha" in response.text.lower() or "api-services-support@amazon.com" in response.text:
+                    print("CAPTCHA or anti-bot measure detected in response")
+                    
+                    if attempt < max_retries:
+                        # Apply a longer delay before the next retry for anti-bot
+                        captcha_delay = delay * 3
+                        print(f"Detected anti-bot measure. Waiting {captcha_delay:.2f}s before retry")
+                        time.sleep(captcha_delay)
+                        continue
+                    
+                    print("Failed to bypass anti-bot measures after all retries")
+                
+                # If everything is good, return the response
+                print(f"Request successful: {url} (Status: {response.status_code})")
+                return response
+
+            except RequestsError as e:
+                print(f"Network error on attempt {attempt+1}: {e}")
+                if attempt == max_retries:
+                    print(f"Max retries reached. Network error: {e}")
+                    return None
+                time.sleep(delay * 2)  # Longer delay after network error
+                
+            except Exception as e:
+                print(f"Unexpected error on attempt {attempt+1}: {e}")
+                if attempt == max_retries:
+                    print(f"Max retries reached. Error: {e}")
+                    return None
+                time.sleep(delay * 2)
+        
+        return None
+        
+    def update_config(self, **kwargs):
+        """
+        Update session configuration parameters.
+        
+        Args:
+            **kwargs: Configuration key-value pairs to update
+        """
+        self.config.update(kwargs)
+        print(f"Updated session configuration: {kwargs}") 

amzpy/utils.py

@@ -35,4 +35,46 @@ def parse_amazon_url(url: str) -> Optional[Tuple[str, str]]:
     # Extract product ID
     product_id = match.group(2)
     
-    return base_url, product_id 
+    return base_url, product_id 
+
+def format_canonical_url(url: str, asin: str, country_code: str = None) -> str:
+    """
+    Format a canonical Amazon product URL in the form amazon.{country}/dp/{asin}
+    
+    Args:
+        url (str): Original Amazon URL
+        asin (str): ASIN of the product
+        country_code (str, optional): Country code (e.g., "com", "in")
+        
+    Returns:
+        str: Canonical URL
+    """
+    if not asin:
+        return url  # Return original if no ASIN available
+        
+    # If country_code is not provided, try to extract it from the original URL
+    if not country_code:
+        try:
+            parsed_url = urlparse(url)
+            domain_parts = parsed_url.netloc.split('.')
+            # Extract country code from domain (e.g., www.amazon.com -> com)
+            if len(domain_parts) >= 3 and 'amazon' in domain_parts:
+                amazon_index = domain_parts.index('amazon')
+                if amazon_index + 1 < len(domain_parts):
+                    country_code = domain_parts[amazon_index + 1]
+        except Exception:
+            country_code = "com"  # Default to .com if extraction fails
+    
+    # Default to .com if still no country code
+    if not country_code:
+        country_code = "com"
+        
+    # Create canonical URL
+    return f"https://www.amazon.{country_code}/dp/{asin}"
+
+# Function to extract brand name from text
+def extract_brand_name(text):
+    match = re.search(r'visit the (.+?) store', text, re.IGNORECASE)
+    if match:
+        return match.group(1).strip()
+    return None