scraprime 0.1.0__tar.gz

scraprime-0.1.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: scraprime
+Version: 0.1.0
+Summary: Prime infrastructure for undetectable scraping. 3-tier WAF bypass & auto-healing parsers.
+Home-page: https://github.com/yourusername/scraprime
+Author: Anees Ur Rahman
+Author-email: anisshewa@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Requires-Dist: curl_cffi>=0.7.0
+Requires-Dist: scrapling>=0.2.0
+Requires-Dist: camoufox[geoip]>=0.4.0
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: selectolax>=0.3.21
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

scraprime-0.1.0/scraprime/__init__.py

@@ -0,0 +1,16 @@
+from .fetcher import StealthFetcher
+from .parser import AdaptiveParser
+from .webhook import send_to_n8n, send_to_n8n_sync
+from .models import ProxyConfig, FetchRequest, ScrapedItem
+
+__all__ = [
+    "StealthFetcher", 
+    "AdaptiveParser", 
+    "send_to_n8n", 
+    "send_to_n8n_sync",
+    "ProxyConfig", 
+    "FetchRequest",
+    "ScrapedItem"
+]
+
+__version__ = "0.1.0"

scraprime-0.1.0/scraprime/fetcher.py

@@ -0,0 +1,162 @@
+import asyncio
+from typing import Optional
+from curl_cffi import requests as cffi_requests
+from scrapling.fetchers import StealthyFetcher as ScraplingFetcher
+
+try:
+    from camoufox.async_api import AsyncCamoufox
+    HAS_CAMOUFOX = True
+except ImportError:
+    HAS_CAMOUFOX = False
+
+class StealthFetcher:
+    """
+    Scraprime's core fetching engine.
+    Implements a 3-tiered fallback system to bypass WAFs (Cloudflare, DataDome, Akamai)
+    while maintaining maximum speed and minimizing resource usage.
+    """
+
+    def __init__(self, proxy: Optional[dict] = None):
+        """
+        Initializes the fetcher with optional proxy configuration.
+        
+        :param proxy: Dictionary containing proxy details. 
+                      Example: {"server": "http://ip:port", "username": "user", "password": "pass"}
+        """
+        self.proxy = proxy
+
+    async def fetch(self, url: str, headers: Optional[dict] = None) -> str:
+        """
+        Main entry point. Attempts to fetch HTML using 3 tiers.
+        Tier 1: Fast HTTP (curl_cffi)
+        Tier 2: Stealth HTTP (scrapling)
+        Tier 3: Full Browser (Camoufox)
+        
+        :return: Raw HTML string if successful.
+        :raises: Exception if all 3 tiers fail.
+        """
+        print(f"[\033[94mScraprime\033[0m] Initiating fetch for: {url}")
+        
+        # TIER 1
+        html_content = await self._fetch_tier1_curl(url, headers)
+        if html_content:
+            return html_content
+            
+        # TIER 2
+        html_content = await self._fetch_tier2_scrapling(url, headers)
+        if html_content:
+            return html_content
+            
+        # TIER 3
+        html_content = await self._fetch_tier3_camoufox(url)
+        if html_content:
+            return html_content
+            
+        raise Exception(f"[\033[91mScraprime\033[0m] All 3 tiers failed to fetch URL: {url}")
+
+    def _is_waf_blocked(self, status_code: int, text: str) -> bool:
+        """Helper to detect WAF blocks based on status codes and HTML signatures."""
+        if status_code in [403, 429, 503]:
+            return True
+        if "challenge-platform" in text or "datadome" in text.lower() or "Access Denied" in text:
+            return True
+        return False
+
+    async def _fetch_tier1_curl(self, url: str, headers: Optional[dict] = None) -> Optional[str]:
+        """
+        Tier 1: Fastest. Uses curl_cffi to impersonate Chrome's TLS/JA3 fingerprint.
+        """
+        print("[\033[94mScraprime\033[0m] Tier 1 (curl_cffi) attempting fast request...")
+        try:
+            response = await asyncio.to_thread(
+                cffi_requests.get,
+                url,
+                impersonate="chrome120",
+                proxies=self.proxy,
+                headers=headers,
+                timeout=15
+            )
+            
+            if self._is_waf_blocked(response.status_code, response.text):
+                print("[\033[93mScraprime\033[0m] Tier 1 detected WAF block. Falling back...")
+                return None
+                
+            print("[\033[92mScraprime\033[0m] Tier 1 successful.")
+            return response.text
+            
+        except Exception as e:
+            print(f"[\033[93mScraprime\033[0m] Tier 1 Exception: {e}. Falling back...")
+            return None
+
+    async def _fetch_tier2_scrapling(self, url: str, headers: Optional[dict] = None) -> Optional[str]:
+        """
+        Tier 2: Medium speed. Uses Scrapling's built-in stealth fetcher to solve basic JS challenges.
+        """
+        print("[\033[94mScraprime\033[0m] Tier 2 (scrapling) attempting stealth request...")
+        try:
+            # Removed timeout=20 because scrapling interprets it as milliseconds
+            response = await asyncio.to_thread(
+                ScraplingFetcher.fetch,
+                url,
+                proxies=self.proxy,
+                headers=headers
+            )
+            
+            if self._is_waf_blocked(response.status, response.body):
+                print("[\033[93mScraprime\033[0m] Tier 2 detected WAF block. Falling back...")
+                return None
+                
+            print("[\033[92mScraprime\033[0m] Tier 2 successful.")
+            return response.body
+            
+        except Exception as e:
+            print(f"[\033[93mScraprime\033[0m] Tier 2 Exception: {e}. Falling back...")
+            return None
+    async def _fetch_tier3_camoufox(self, url: str) -> Optional[str]:
+        """
+        Tier 3: Slowest but most powerful. Full C++ spoofed Firefox browser for heavy WAFs.
+        """
+        if not HAS_CAMOUFOX:
+            print("[\033[91mScraprime\033[0m] Camoufox is not installed. Cannot use Tier 3.")
+            return None
+
+        print("[\033[94mScraprime\033[0m] Tier 3 (camoufox) launching full browser...")
+        
+        async with AsyncCamoufox(
+            headless=True, 
+            humanize=True,        # Simulate human mouse movements
+            geoip=True,           # Match browser timezone/location to proxy IP
+            block_images=False,   # WAFs flag browsers that don't load images
+            os=['windows'],       # Spoof Windows hardware
+            proxy=self.proxy
+        ) as browser:
+            page = await browser.new_page()
+            try:
+                await page.goto(url, wait_until="networkidle", timeout=45000)
+                
+                # Wait extra time for invisible JS challenges (like DataDome) to solve
+                await page.wait_for_timeout(5000)
+                
+                html = await page.content()
+                
+                if self._is_waf_blocked(200, html):
+                    print("[\033[91mScraprime\033[0m] Tier 3 still blocked. Giving up.")
+                    return None
+                    
+                print("[\033[92mScraprime\033[0m] Tier 3 successful.")
+                return html
+                
+            except Exception as e:
+                print(f"[\033[91mScraprime\033[0m] Tier 3 Browser Error: {e}")
+                return None
+            finally:
+                await browser.close()
+
+# ==========================================
+# USAGE EXAMPLE (Sync wrapper for easy testing)
+# ==========================================
+def fetch_sync(url: str, proxy: Optional[dict] = None) -> str:
+    """Synchronous convenience function for users who don't want to use asyncio.run()."""
+    fetcher = StealthFetcher(proxy=proxy)
+    loop = asyncio.get_event_loop()
+    return loop.run_until_complete(fetcher.fetch(url))

scraprime-0.1.0/scraprime/models.py

@@ -0,0 +1,51 @@
+from typing import Optional, Dict, Any
+from pydantic import BaseModel, HttpUrl, Field
+
+class ProxyConfig(BaseModel):
+    """
+    Pydantic model for configuring proxies.
+    Ensures the user provides the correct structure for curl_cffi and Camoufox.
+    """
+    server: str = Field(..., description="Proxy server address (e.g., http://gateway.ip:port)")
+    username: Optional[str] = None
+    password: Optional[str] = None
+
+    def to_dict(self) -> Dict[str, str]:
+        """Converts the model to the dictionary format expected by curl_cffi/camoufox."""
+        proxy_dict = {"server": self.server}
+        if self.username:
+            proxy_dict["username"] = self.username
+        if self.password:
+            proxy_dict["password"] = self.password
+        return proxy_dict
+
+class FetchRequest(BaseModel):
+    """
+    Pydantic model for orchestrating a complete Scraprime fetch + parse + webhook flow.
+    """
+    url: HttpUrl = Field(..., description="The target URL to scrape.")
+    proxy: Optional[ProxyConfig] = None
+    headers: Optional[Dict[str, str]] = None
+    webhook_url: Optional[HttpUrl] = Field(None, description="Optional n8n/Make.com webhook URL to send data to.")
+
+class ScrapedItem(BaseModel):
+    """
+    Pydantic model for standardizing the output data before sending to webhooks.
+    """
+    url: str
+    data: Dict[str, Any] = Field(default_factory=dict, description="The extracted data payload.")
+
+# ==========================================
+# USAGE EXAMPLE
+# ==========================================
+if __name__ == "__main__":
+    # Example of how a user would validate their config before running
+    try:
+        config = FetchRequest(
+            url="https://www.very.co.uk/product",
+            proxy=ProxyConfig(server="http://1.2.3.4:8080", username="user", password="pass")
+        )
+        print("Configuration valid!")
+        print(f"Proxy dict for fetcher: {config.proxy.to_dict()}")
+    except Exception as e:
+        print(f"Configuration invalid: {e}")

scraprime-0.1.0/scraprime/parser.py

@@ -0,0 +1,93 @@
+import re
+from typing import Optional
+from selectolax.parser import HTMLParser
+
+class AdaptiveParser:
+    """
+    Scraprime's parsing engine.
+    Uses Selectolax for fast DOM traversal and regex for auto-healing data extraction.
+    """
+
+    def __init__(self, html: str):
+        """
+        Initializes the parser with raw HTML.
+        
+        :param html: Raw HTML string fetched by StealthFetcher.
+        """
+        self.tree = HTMLParser(html)
+
+    def extract_meta(self, property_name: str) -> Optional[str]:
+        """
+        Extracts content from <meta> tags. Highly reliable for prices and titles.
+        Example: <meta property="product:price:amount" content="479.00">
+        
+        :param property_name: The property attribute value to search for.
+        :return: The content attribute value or None.
+        """
+        element = self.tree.css_first(f'meta[property="{property_name}"]')
+        if element:
+            return element.attrib.get('content')
+        return None
+
+    def extract_css(self, selector: str) -> Optional[str]:
+        """
+        Standard CSS selector extraction. 
+        
+        :param selector: CSS selector string (e.g., 'h1.product-title').
+        :return: The text of the first matching element or None.
+        """
+        element = self.tree.css_first(selector)
+        if element:
+            return element.text(strip=True)
+        return None
+
+    def extract_nearby_regex(self, anchor_text: str, pattern: str) -> Optional[str]:
+        """
+        Auto-Healing Tool. Searches the entire page text for a regex pattern.
+        If an anchor is provided, it tries to find the pattern near that text.
+        
+        :param anchor_text: Text to find the starting point (e.g., "Add to Cart").
+        :param pattern: Regex pattern to search for (e.g., r'[\£\$\€]\d+[.,]\d{2}').
+        :return: The matched string or None.
+        """
+        # Get all visible text on the page
+        page_text = self.tree.text()
+        
+        # Find all matches in the entire page
+        matches = list(re.finditer(pattern, page_text))
+        if not matches:
+            return None
+            
+        # Try to find a match close to the anchor text
+        anchor_idx = page_text.find(anchor_text)
+        if anchor_idx != -1:
+            # Find the match with the smallest distance to the anchor
+            closest_match = min(matches, key=lambda m: abs(m.start() - anchor_idx))
+            return closest_match.group(0).strip()
+            
+        # If anchor not found, just return the first match found on the page
+        return matches[0].group(0).strip()
+
+# ==========================================
+# USAGE EXAMPLE
+# ==========================================
+if __name__ == "__main__":
+    mock_html = """
+    <html>
+        <head>
+            <meta property="og:title" content="Narva Sofa" />
+            <meta property="product:price:amount" content="479.00" />
+        </head>
+        <body>
+            <h1 class="title">Narva Fabric 2 Seater Sofa</h1>
+            <div class="price">£479.00</div>
+            <button>Add to Basket</button>
+        </body>
+    </html>
+    """
+    
+    parser = AdaptiveParser(mock_html)
+    
+    print(f"Title (Meta): {parser.extract_meta('og:title')}")
+    print(f"Price (Meta): {parser.extract_meta('product:price:amount')}")
+    print(f"Price (Nearby): {parser.extract_nearby_regex('Add to Basket', r'[\£\$\€]\d+[.,]\d{2}')}")

scraprime-0.1.0/scraprime/webhook.py

@@ -0,0 +1,71 @@
+import asyncio
+import aiohttp
+import json
+from typing import Optional, Dict, Any
+
+async def send_to_n8n(webhook_url: str, payload: Dict[str, Any]) -> bool:
+    """
+    Asynchronously sends a JSON payload to an n8n, Make.com, or Zapier webhook.
+    
+    :param webhook_url: The webhook URL provided by the automation tool.
+    :param payload: A dictionary containing the scraped data.
+    :return: True if successful, False otherwise.
+    """
+    if not webhook_url:
+        return False
+        
+    print(f"[\033[94mScraprime\033[0m] Dispatching data to webhook: {webhook_url}")
+    
+    try:
+        # Use aiohttp for non-blocking HTTP POST request
+        async with aiohttp.ClientSession() as session:
+            headers = {"Content-Type": "application/json"}
+            async with session.post(webhook_url, json=payload, headers=headers, timeout=10) as response:
+                
+                if response.status in [200, 201, 202]:
+                    print("[\033[92mScraprime\033[0m] Webhook successfully received.")
+                    return True
+                else:
+                    text = await response.text()
+                    print(f"[\033[91mScraprime\033[0m] Webhook failed. Status: {response.status}, Response: {text}")
+                    return False
+                    
+    except asyncio.TimeoutError:
+        print("[\033[91mScraprime\033[0m] Webhook timed out.")
+        return False
+    except Exception as e:
+        print(f"[\033[91mScraprime\033[0m] Webhook Exception: {e}")
+        return False
+
+def send_to_n8n_sync(webhook_url: str, payload: Dict[str, Any]) -> bool:
+    """
+    Synchronous wrapper for send_to_n8n. 
+    Useful for users who are not running an async event loop.
+    """
+    loop = asyncio.get_event_loop()
+    return loop.run_until_complete(send_to_n8n(webhook_url, payload))
+
+
+# ==========================================
+# USAGE EXAMPLE
+# ==========================================
+if __name__ == "__main__":
+    # Simulated scraped data
+    scraped_data = {
+        "url": "https://www.very.co.uk/narva-sofa",
+        "product_name": "Narva Fabric 2 Seater Sofa",
+        "price": "479.00",
+        "stock_status": "In Stock"
+    }
+    
+    # Put your actual n8n webhook URL here to test it!
+    # n8n webhooks usually look like: https://your-n8n.app/webhook/xyz-123
+    test_webhook_url = "https://webhook.site/your-unique-id" 
+    
+    print("Testing sync webhook sender...")
+    success = send_to_n8n_sync(test_webhook_url, scraped_data)
+    
+    if success:
+        print("Check your webhook tool! The data should be there.")
+    else:
+        print("Webhook failed. Check the URL or network.")

scraprime-0.1.0/scraprime.egg-info/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: scraprime
+Version: 0.1.0
+Summary: Prime infrastructure for undetectable scraping. 3-tier WAF bypass & auto-healing parsers.
+Home-page: https://github.com/yourusername/scraprime
+Author: Anees Ur Rahman
+Author-email: anisshewa@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Requires-Dist: curl_cffi>=0.7.0
+Requires-Dist: scrapling>=0.2.0
+Requires-Dist: camoufox[geoip]>=0.4.0
+Requires-Dist: aiohttp>=3.9.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: selectolax>=0.3.21
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

scraprime-0.1.0/scraprime.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+setup.py
+scraprime/__init__.py
+scraprime/fetcher.py
+scraprime/models.py
+scraprime/parser.py
+scraprime/webhook.py
+scraprime.egg-info/PKG-INFO
+scraprime.egg-info/SOURCES.txt
+scraprime.egg-info/dependency_links.txt
+scraprime.egg-info/requires.txt
+scraprime.egg-info/top_level.txt

scraprime-0.1.0/scraprime.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

scraprime-0.1.0/scraprime.egg-info/requires.txt

@@ -0,0 +1,6 @@
+curl_cffi>=0.7.0
+scrapling>=0.2.0
+camoufox[geoip]>=0.4.0
+aiohttp>=3.9.0
+pydantic>=2.0.0
+selectolax>=0.3.21

scraprime-0.1.0/scraprime.egg-info/top_level.txt

@@ -0,0 +1 @@
+scraprime

scraprime-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

scraprime-0.1.0/setup.py

@@ -0,0 +1,25 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="scraprime",
+    version="0.1.0",
+    description="Prime infrastructure for undetectable scraping. 3-tier WAF bypass & auto-healing parsers.",
+    author="Anees Ur Rahman",  
+    author_email="anisshewa@gmail.com",
+    url="https://github.com/yourusername/scraprime",
+    packages=find_packages(),
+    install_requires=[
+        "curl_cffi>=0.7.0",
+        "scrapling>=0.2.0",
+        "camoufox[geoip]>=0.4.0",  # <-- This ensures geoip installs automatically!
+        "aiohttp>=3.9.0",
+        "pydantic>=2.0.0",
+        "selectolax>=0.3.21",
+    ],
+    python_requires=">=3.10",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+)