askpablos-scrapy-api 0.1.0__tar.gz

askpablos_scrapy_api-0.1.0/PKG-INFO

@@ -0,0 +1,103 @@
+Metadata-Version: 2.4
+Name: askpablos-scrapy-api
+Version: 0.1.0
+Summary: Scrapy middleware to route requests through AskPablos Proxy API
+Author-email: Fawad Ali <fawadstar6@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/fawadss1/askpablos-scrapy-api
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: scrapy>=2.6.0
+Requires-Dist: requests>=2.25.0
+
+# AskPablos Scrapy API Middleware
+
+**AskPablosScrapyAPI** is a Scrapy downloader middleware that allows your spiders to transparently route specific requests through an external proxy API like AskPablos. It supports headless browser mode and rotating proxies on a per-request basis, while integrating cleanly with Scrapy's native settings system.
+
+---
+
+## 🚀 Features
+
+- ✅ Only activates for requests that explicitly ask for proxy routing
+- ✅ Supports rotating proxies and headless browser rendering
+- ✅ Compatible with per-spider `CUSTOM_SETTINGS`
+- ✅ Automatically signs requests using HMAC
+- ✅ Clean plug-and-play design for reuse across Scrapy projects
+
+---
+
+## 📦 Installation
+
+Install via pip:
+
+```bash
+pip install askpablos-scrapy-api
+```
+
+Or directly from the repository:
+
+```bash
+pip install git+https://github.com/fawadss1/askpablos-scrapy-api.git
+```
+
+---
+
+## 🔧 Quick Setup
+
+1. Add the middleware to your Scrapy project settings:
+
+```python
+# settings.py
+DOWNLOADER_MIDDLEWARES = {
+    "askpablos_scrapy_api.AskPablosScrapyAPI": 543,
+}
+
+# API credentials
+API_KEY = "your-api-key"
+SECRET_KEY = "your-secret-key"
+```
+
+2. Use in your spider by adding `askpablos_api_map` to the request meta:
+
+```python
+def start_requests(self):
+    yield scrapy.Request(
+        url="https://example.com",
+        callback=self.parse,
+        meta={
+            "askpablos_api_map": {
+                "browser": True,        # Use headless browser
+                "rotate_proxy": True    # Use rotating proxy IP
+            }
+        }
+    )
+```
+
+---
+
+## 📚 Documentation
+
+For detailed usage instructions and advanced configurations:
+
+- [Usage Guide](https://github.com/fawadss1/askpablos-scrapy-api/blob/main/docs/usage.md)
+- [FAQ](https://github.com/fawadss1/askpablos-scrapy-api/blob/main/docs/faq.md)
+
+---
+
+## 📋 Requirements
+
+- Python 3.7+
+- Scrapy 2.6+
+- Valid AskPablos Proxy API credentials
+
+---
+
+## 📝 License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+---
+
+## 👤 Author
+
+Fawad Ali ([@fawadss1](https://github.com/fawadss1))

askpablos_scrapy_api-0.1.0/README.md

@@ -0,0 +1,91 @@
+# AskPablos Scrapy API Middleware
+
+**AskPablosScrapyAPI** is a Scrapy downloader middleware that allows your spiders to transparently route specific requests through an external proxy API like AskPablos. It supports headless browser mode and rotating proxies on a per-request basis, while integrating cleanly with Scrapy's native settings system.
+
+---
+
+## 🚀 Features
+
+- ✅ Only activates for requests that explicitly ask for proxy routing
+- ✅ Supports rotating proxies and headless browser rendering
+- ✅ Compatible with per-spider `CUSTOM_SETTINGS`
+- ✅ Automatically signs requests using HMAC
+- ✅ Clean plug-and-play design for reuse across Scrapy projects
+
+---
+
+## 📦 Installation
+
+Install via pip:
+
+```bash
+pip install askpablos-scrapy-api
+```
+
+Or directly from the repository:
+
+```bash
+pip install git+https://github.com/fawadss1/askpablos-scrapy-api.git
+```
+
+---
+
+## 🔧 Quick Setup
+
+1. Add the middleware to your Scrapy project settings:
+
+```python
+# settings.py
+DOWNLOADER_MIDDLEWARES = {
+    "askpablos_scrapy_api.AskPablosScrapyAPI": 543,
+}
+
+# API credentials
+API_KEY = "your-api-key"
+SECRET_KEY = "your-secret-key"
+```
+
+2. Use in your spider by adding `askpablos_api_map` to the request meta:
+
+```python
+def start_requests(self):
+    yield scrapy.Request(
+        url="https://example.com",
+        callback=self.parse,
+        meta={
+            "askpablos_api_map": {
+                "browser": True,        # Use headless browser
+                "rotate_proxy": True    # Use rotating proxy IP
+            }
+        }
+    )
+```
+
+---
+
+## 📚 Documentation
+
+For detailed usage instructions and advanced configurations:
+
+- [Usage Guide](https://github.com/fawadss1/askpablos-scrapy-api/blob/main/docs/usage.md)
+- [FAQ](https://github.com/fawadss1/askpablos-scrapy-api/blob/main/docs/faq.md)
+
+---
+
+## 📋 Requirements
+
+- Python 3.7+
+- Scrapy 2.6+
+- Valid AskPablos Proxy API credentials
+
+---
+
+## 📝 License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+---
+
+## 👤 Author
+
+Fawad Ali ([@fawadss1](https://github.com/fawadss1))

askpablos_scrapy_api-0.1.0/askpablos_scrapy_api/__init__.py

@@ -0,0 +1 @@
+from .middleware import AskPablosScrapyAPI

askpablos_scrapy_api-0.1.0/askpablos_scrapy_api/middleware.py

@@ -0,0 +1,98 @@
+import json
+import base64
+import hmac
+import hashlib
+import requests
+from scrapy.http import HtmlResponse
+
+
+class AskPablosScrapyAPI:
+    """
+    Scrapy middleware to route selected requests through AskPablos proxy API.
+
+    This middleware activates **only** for requests that include:
+        meta = {
+            "askpablos_api_map": {
+                "browser": True,          # Optional: Use headless browser
+                "rotate_proxy": True      # Optional: Use rotating proxy IP
+            }
+        }
+
+    It will bypass any request that does not include the `askpablos_api_map` key or has it as an empty dict.
+
+    Configuration (via settings.py or `CUSTOM_SETTINGS`):
+        API_KEY      = "<your API key>"
+        SECRET_KEY   = "<your secret key>"
+    """
+
+    API_URL = "http://10.10.10.178:7500/api/proxy/"
+
+    def __init__(self, api_key, secret_key):
+        self.api_key = api_key
+        self.secret_key = secret_key
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        api_key = crawler.settings.get("API_KEY")
+        secret_key = crawler.settings.get("SECRET_KEY")
+
+        if not api_key or not secret_key:
+            raise ValueError("API_KEY and SECRET_KEY must be defined in settings.")
+
+        return cls(api_key=api_key, secret_key=secret_key)
+
+    def process_request(self, request, spider):
+        proxy_cfg = request.meta.get("askpablos_api_map")
+
+        if not proxy_cfg or not isinstance(proxy_cfg, dict) or not proxy_cfg:
+            return None  # Skip proxying
+
+        browser = proxy_cfg.get("browser", False)
+        rotate_proxy = proxy_cfg.get("rotate_proxy", False)
+
+        payload = {
+            "url": request.url,
+            "method": "GET",
+            "browser": browser,
+            "rotateProxy": rotate_proxy
+        }
+
+        try:
+            request_json = json.dumps(payload, separators=(',', ':'))
+            signature = hmac.new(
+                self.secret_key.encode(),
+                request_json.encode(),
+                hashlib.sha256
+            ).digest()
+            signature_b64 = base64.b64encode(signature).decode()
+
+            headers = {
+                "Content-Type": "application/json",
+                "X-API-Key": self.api_key,
+                "X-Signature": signature_b64
+            }
+
+            response = requests.post(self.API_URL, data=request_json, headers=headers, timeout=30)
+            response.raise_for_status()
+
+            try:
+                proxy_response = response.json()
+            except ValueError:
+                spider.logger.error(f"[AskPablos API] Invalid JSON response from {self.API_URL}")
+                return None
+
+            html_body = proxy_response.get("body")
+            if not html_body:
+                spider.logger.error(f"[AskPablos API] No 'body' in response for {request.url}")
+                return None
+
+            return HtmlResponse(
+                url=request.url,
+                body=html_body,
+                encoding="utf-8",
+                request=request
+            )
+
+        except Exception as e:
+            spider.logger.error(f"[AskPablos API] Error processing {request.url}: {e}")
+            return None

askpablos_scrapy_api-0.1.0/askpablos_scrapy_api.egg-info/PKG-INFO

@@ -0,0 +1,103 @@
+Metadata-Version: 2.4
+Name: askpablos-scrapy-api
+Version: 0.1.0
+Summary: Scrapy middleware to route requests through AskPablos Proxy API
+Author-email: Fawad Ali <fawadstar6@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/fawadss1/askpablos-scrapy-api
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: scrapy>=2.6.0
+Requires-Dist: requests>=2.25.0
+
+# AskPablos Scrapy API Middleware
+
+**AskPablosScrapyAPI** is a Scrapy downloader middleware that allows your spiders to transparently route specific requests through an external proxy API like AskPablos. It supports headless browser mode and rotating proxies on a per-request basis, while integrating cleanly with Scrapy's native settings system.
+
+---
+
+## 🚀 Features
+
+- ✅ Only activates for requests that explicitly ask for proxy routing
+- ✅ Supports rotating proxies and headless browser rendering
+- ✅ Compatible with per-spider `CUSTOM_SETTINGS`
+- ✅ Automatically signs requests using HMAC
+- ✅ Clean plug-and-play design for reuse across Scrapy projects
+
+---
+
+## 📦 Installation
+
+Install via pip:
+
+```bash
+pip install askpablos-scrapy-api
+```
+
+Or directly from the repository:
+
+```bash
+pip install git+https://github.com/fawadss1/askpablos-scrapy-api.git
+```
+
+---
+
+## 🔧 Quick Setup
+
+1. Add the middleware to your Scrapy project settings:
+
+```python
+# settings.py
+DOWNLOADER_MIDDLEWARES = {
+    "askpablos_scrapy_api.AskPablosScrapyAPI": 543,
+}
+
+# API credentials
+API_KEY = "your-api-key"
+SECRET_KEY = "your-secret-key"
+```
+
+2. Use in your spider by adding `askpablos_api_map` to the request meta:
+
+```python
+def start_requests(self):
+    yield scrapy.Request(
+        url="https://example.com",
+        callback=self.parse,
+        meta={
+            "askpablos_api_map": {
+                "browser": True,        # Use headless browser
+                "rotate_proxy": True    # Use rotating proxy IP
+            }
+        }
+    )
+```
+
+---
+
+## 📚 Documentation
+
+For detailed usage instructions and advanced configurations:
+
+- [Usage Guide](https://github.com/fawadss1/askpablos-scrapy-api/blob/main/docs/usage.md)
+- [FAQ](https://github.com/fawadss1/askpablos-scrapy-api/blob/main/docs/faq.md)
+
+---
+
+## 📋 Requirements
+
+- Python 3.7+
+- Scrapy 2.6+
+- Valid AskPablos Proxy API credentials
+
+---
+
+## 📝 License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+---
+
+## 👤 Author
+
+Fawad Ali ([@fawadss1](https://github.com/fawadss1))

askpablos_scrapy_api-0.1.0/askpablos_scrapy_api.egg-info/SOURCES.txt

@@ -0,0 +1,9 @@
+README.md
+pyproject.toml
+askpablos_scrapy_api/__init__.py
+askpablos_scrapy_api/middleware.py
+askpablos_scrapy_api.egg-info/PKG-INFO
+askpablos_scrapy_api.egg-info/SOURCES.txt
+askpablos_scrapy_api.egg-info/dependency_links.txt
+askpablos_scrapy_api.egg-info/requires.txt
+askpablos_scrapy_api.egg-info/top_level.txt

askpablos_scrapy_api-0.1.0/askpablos_scrapy_api.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

askpablos_scrapy_api-0.1.0/askpablos_scrapy_api.egg-info/requires.txt

@@ -0,0 +1,2 @@
+scrapy>=2.6.0
+requests>=2.25.0

askpablos_scrapy_api-0.1.0/askpablos_scrapy_api.egg-info/top_level.txt

@@ -0,0 +1 @@
+askpablos_scrapy_api

askpablos_scrapy_api-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "askpablos-scrapy-api"
+version = "0.1.0"
+description = "Scrapy middleware to route requests through AskPablos Proxy API"
+authors = [
+    { name="Fawad Ali", email="fawadstar6@gmail.com" }
+]
+license = {text = "MIT"}
+readme = "README.md"
+requires-python = ">=3.7"
+dependencies = [
+    "scrapy>=2.6.0",
+    "requests>=2.25.0"
+]
+
+[project.urls]
+Homepage = "https://github.com/fawadss1/askpablos-scrapy-api"
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"

askpablos_scrapy_api-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+