crawlerflow 0.2.0__tar.gz → 0.3.0__tar.gz

{crawlerflow-0.2.0 → crawlerflow-0.3.0}/CHANGELOG.md

@@ -5,6 +5,28 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and this project
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.0] - 2026-09-17
+
+### Added
+
+- Workflow-wide proxy support. `WorkflowContext` carries a `proxy_url` that survives parallel loop
+  forks, and the built-in `http_request`, `resolve_location_url`, `enrich_html_links_http`, and
+  `enrich_json_map_locations` steps route their requests through it.
+- `BrowserAdapter.configure_proxy()` extension hook. Adapters that cannot proxy raise a clear
+  error, keeping the contract explicit.
+- Pydoll proxy support for HTTP, HTTPS, and SOCKS5 URLs, including authenticated proxies through a
+  generated extension. The temporary extension directory is removed when the adapter closes.
+- CapSolver reference plugin that solves captcha tasks and exposes the solution to later steps.
+- Webshare reference plugin that selects proxies and publishes the active proxy to the workflow.
+- Combined CapSolver and Webshare example workflows covering browser and HTTP-only Cloudflare
+  challenge flows.
+- Plugin documentation covering proxy-aware plugins and the new adapter hook.
+
+### Fixed
+
+- Disabled Jekyll processing for the published documentation site so workflow expression examples
+  are no longer parsed as Liquid templates.
+
 ## [0.2.0] - 2026-08-18
 
 ### Changed
@@ -36,5 +58,6 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - Plugin API with typed settings, lifecycle hooks, steps, filters, and subscribers.
 - `run`, `validate`, `list-steps`, `list-plugins`, and `doctor` CLI commands.
 
+[0.3.0]: https://github.com/mehmetemineker/crawlerflow/releases/tag/v0.3.0
 [0.2.0]: https://github.com/mehmetemineker/crawlerflow/releases/tag/v0.2.0
 [0.1.0]: https://github.com/mehmetemineker/crawlerflow/releases/tag/v0.1.0

{crawlerflow-0.2.0 → crawlerflow-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: crawlerflow
-Version: 0.2.0
+Version: 0.3.0
 Summary: Declarative YAML workflow engine for browser automation and web scraping
 Project-URL: Homepage, https://github.com/mehmetemineker/crawlerflow
 Project-URL: Repository, https://github.com/mehmetemineker/crawlerflow

{crawlerflow-0.2.0 → crawlerflow-0.3.0}/crawlerflow/__init__.py

@@ -5,5 +5,5 @@ from __future__ import annotations
 from crawlerflow.engine.runner import WorkflowRunner
 
 __all__ = ["WorkflowRunner"]
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 

{crawlerflow-0.2.0 → crawlerflow-0.3.0}/crawlerflow/browser/base.py

@@ -66,6 +66,13 @@ class BrowserAdapter(ABC):
     @abstractmethod
     async def screenshot(self, path: Path) -> Path: ...
 
+    def configure_proxy(self, proxy_url: str) -> None:
+        """Configure a proxy before the browser session starts, if supported."""
+
+        raise RuntimeError(
+            f"Browser adapter {type(self).__name__} does not support proxy configuration"
+        )
+
     async def close(self) -> None:
         """Release browser resources when an adapter owns them."""
 

{crawlerflow-0.2.0 → crawlerflow-0.3.0}/crawlerflow/browser/pydoll.py

@@ -6,11 +6,14 @@ import asyncio
 import json
 import logging
 import math
+import shutil
+import tempfile
 import time
 from collections.abc import Mapping
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Any
+from urllib.parse import unquote, urlsplit
 
 from crawlerflow.browser.base import BrowserAdapter, BrowserResponse
 
@@ -40,6 +43,7 @@ class PydollBrowserConfig:
     default_wait_timeout: float = 10
     network_idle_period: float = 0.5
     download_directory: Path | None = None
+    proxy_url: str | None = None
 
 
 class PydollBrowserAdapter(BrowserAdapter):
@@ -61,6 +65,7 @@ class PydollBrowserAdapter(BrowserAdapter):
         self._network_callback_ids: list[int] = []
         self._inflight_requests: set[str] = set()
         self._last_network_activity = time.monotonic()
+        self._proxy_extension_dir: Path | None = None
 
     async def goto(self, url: str) -> None:
         tab = await self._get_tab()
@@ -174,6 +179,18 @@ class PydollBrowserAdapter(BrowserAdapter):
         await tab.take_screenshot(path)
         return path
 
+    def configure_proxy(self, proxy_url: str) -> None:
+        """Set an HTTP(S)/SOCKS5 proxy before Chromium is launched."""
+
+        if self._browser is not None or self._tab is not None:
+            raise PydollAdapterError("Proxy must be configured before the browser starts")
+        parsed = urlsplit(proxy_url)
+        if parsed.scheme not in {"http", "https", "socks5"} or not parsed.hostname:
+            raise PydollAdapterError("Proxy URL must include an HTTP(S) or SOCKS5 host")
+        if parsed.port is None or not 1 <= parsed.port <= 65535:
+            raise PydollAdapterError("Proxy URL must include a valid port")
+        self.config = replace(self.config, proxy_url=proxy_url)
+
     async def close(self) -> None:
         if self._tab is not None:
             for callback_id in self._network_callback_ids:
@@ -191,6 +208,9 @@ class PydollBrowserAdapter(BrowserAdapter):
         self._browser = None
         self._network_tracking_enabled = False
         self._inflight_requests.clear()
+        if self._proxy_extension_dir is not None:
+            shutil.rmtree(self._proxy_extension_dir, ignore_errors=True)
+            self._proxy_extension_dir = None
 
     async def _get_tab(self) -> Any:
         if self._tab is None:
@@ -220,10 +240,51 @@ class PydollBrowserAdapter(BrowserAdapter):
                 options.set_default_download_directory(str(self.config.download_directory))
             for argument in self.config.arguments:
                 options.add_argument(argument)
+            if self.config.proxy_url is not None:
+                self._configure_proxy_options(options, self.config.proxy_url)
 
             self._browser = Chrome(options=options)
         self._tab = await self._browser.start()
 
+    def _configure_proxy_options(self, options: Any, proxy_url: str) -> None:
+        parsed = urlsplit(proxy_url)
+        host = parsed.hostname
+        port = parsed.port
+        if host is None or port is None:
+            raise PydollAdapterError("Proxy URL must include a host and port")
+        options.add_argument(f"--proxy-server={parsed.scheme}://{host}:{port}")
+        if parsed.username is not None or parsed.password is not None:
+            if parsed.username is None or parsed.password is None:
+                raise PydollAdapterError("Proxy URL must include both username and password")
+            self._proxy_extension_dir = self._create_proxy_auth_extension(
+                unquote(parsed.username),
+                unquote(parsed.password),
+            )
+            options.add_argument(f"--load-extension={self._proxy_extension_dir}")
+
+    @staticmethod
+    def _create_proxy_auth_extension(username: str, password: str) -> Path:
+        extension_dir = Path(tempfile.mkdtemp(prefix="crawlerflow-proxy-auth-"))
+        manifest = {
+            "manifest_version": 2,
+            "name": "Crawlerflow proxy authentication",
+            "version": "1.0",
+            "permissions": ["webRequest", "webRequestBlocking", "<all_urls>"],
+            "background": {"scripts": ["background.js"]},
+        }
+        background = (
+            "chrome.webRequest.onAuthRequired.addListener("
+            "function(details) { return {authCredentials: {username: "
+            f"{json.dumps(username)}, password: {json.dumps(password)}}}; }}, "
+            "{urls: ['<all_urls>']}, ['blocking']);"
+        )
+        (extension_dir / "manifest.json").write_text(
+            json.dumps(manifest),
+            encoding="utf-8",
+        )
+        (extension_dir / "background.js").write_text(background, encoding="utf-8")
+        return extension_dir
+
     async def _query(self, selector: str, timeout_seconds: float | None = None) -> Any:
         tab = await self._get_tab()
         timeout = timeout_seconds or self.config.default_wait_timeout

{crawlerflow-0.2.0 → crawlerflow-0.3.0}/crawlerflow/engine/context.py

@@ -22,6 +22,7 @@ class WorkflowContext:
     workflow_name: str
     base_path: Path
     browser: BrowserAdapter | None = None
+    proxy_url: str | None = None
     variables: dict[str, Any] = field(default_factory=dict)
     outputs: dict[str, Any] = field(default_factory=dict)
     cookies: dict[str, str] = field(default_factory=dict)
@@ -51,6 +52,7 @@ class WorkflowContext:
             workflow_name=self.workflow_name,
             base_path=self.base_path,
             browser=self.browser,
+            proxy_url=self.proxy_url,
             variables=dict(self.variables),
             outputs=dict(self.outputs),
             cookies=dict(self.cookies),

{crawlerflow-0.2.0 → crawlerflow-0.3.0}/crawlerflow/steps/utility.py

@@ -939,7 +939,10 @@ class EnrichHtmlLinksHttpStep(BaseStep[EnrichHtmlLinksHttpConfig]):
         cache: dict[str, str | None] = {}
         insertions: dict[int, list[tuple[str, str]]] = {}
         last_finished_at: float | None = None
-        async with httpx.AsyncClient(timeout=self.config.timeout) as client:
+        async with httpx.AsyncClient(
+            timeout=self.config.timeout,
+            proxy=context.proxy_url,
+        ) as client:
             for target_index, url in collector.links:
                 if url not in cache:
                     if last_finished_at is not None and self.config.delay > 0:
@@ -1092,6 +1095,7 @@ class ResolveLocationUrlStep(BaseStep[ResolveLocationUrlConfig]):
                 timeout=self.config.timeout,
                 follow_redirects=True,
                 max_redirects=self.config.max_redirects,
+                proxy=context.proxy_url,
             ) as client:
                 response = await client.request(
                     "GET",
@@ -1307,6 +1311,7 @@ class EnrichJsonMapLocationsStep(BaseStep[EnrichJsonMapLocationsConfig]):
             timeout=self.config.timeout,
             follow_redirects=True,
             max_redirects=self.config.max_redirects,
+            proxy=context.proxy_url,
         ) as client:
 
             async def resolve(url: str) -> tuple[str, dict[str, Any] | None]:
@@ -1435,7 +1440,10 @@ class HttpRequestStep(BaseStep[HttpRequestConfig]):
             {"transport": "http", "method": method, "url": self.config.url},
         )
         try:
-            async with httpx.AsyncClient(timeout=self.config.timeout) as client:
+            async with httpx.AsyncClient(
+                timeout=self.config.timeout,
+                proxy=context.proxy_url,
+            ) as client:
                 response = await client.request(
                     method,
                     self.config.url,

{crawlerflow-0.2.0 → crawlerflow-0.3.0}/docs/plugins.md

@@ -120,6 +120,56 @@ python -m crawlerflow list-plugins
 
 The command displays each plugin name, import target, and owning Python distribution.
 
+## Webshare proxy plugin
+
+The repository also contains a separately installable `crawlerflow-webshare` plugin. It reads the
+Webshare proxy list, selects one valid proxy randomly during workflow startup, and keeps that proxy
+for the workflow's browser and direct HTTP requests:
+
+```bash
+python -m pip install -e . -e examples/plugins/webshare
+export WEBSHARE_API_KEY="your-webshare-api-key"
+```
+
+```yaml
+plugins:
+  - name: webshare
+    settings:
+      mode: direct
+      country_codes: [US]
+      valid_only: true
+      proxy_retry:
+        enabled: true
+        attempts_per_proxy: 3
+        max_proxies: 5
+```
+
+See the complete example in
+[`examples/plugins/webshare/workflow.yaml`](https://github.com/mehmetemineker/crawlerflow/blob/main/examples/plugins/webshare/workflow.yaml).
+
+For an HTTP-only variant without Pydoll, use
+[`cloudflare-challenge-http.yaml`](https://github.com/mehmetemineker/crawlerflow/blob/main/examples/plugins/capsolver-webshare/cloudflare-challenge-http.yaml).
+It fetches the challenge HTML and sends the returned `cf_clearance` cookie with a second direct
+HTTP request through the same proxy.
+
+## Cloudflare Challenge example
+
+The combined CapSolver and Webshare example uses CapSolver's `AntiCloudflareTask`. It opens the
+target with a fixed user agent, passes the same selected Webshare proxy to CapSolver, applies the
+returned clearance cookies, and reloads the target page:
+
+```bash
+python -m pip install -e . -e examples/plugins/capsolver -e examples/plugins/webshare
+export CAPSOLVER_API_KEY="your-capsolver-api-key"
+export WEBSHARE_API_KEY="your-webshare-api-key"
+python -m crawlerflow run examples/plugins/capsolver-webshare/cloudflare-challenge.yaml
+```
+
+Replace the authorized target URL before running. Because the example enables
+`include_credentials: true` to pass the proxy to CapSolver, protect the generated result file;
+it can contain a `cf_clearance` cookie. See
+[`cloudflare-challenge.yaml`](https://github.com/mehmetemineker/crawlerflow/blob/main/examples/plugins/capsolver-webshare/cloudflare-challenge.yaml).
+
 ## Example package
 
 `examples/plugins/example` is a separately installable reference package. It registers a custom

crawlerflow-0.3.0/examples/plugins/capsolver/README.md

@@ -0,0 +1,66 @@
+# Crawlerflow CapSolver plugin
+
+This separately installable plugin exposes CapSolver's generic task API to Crawlerflow. The task
+object is passed through without a task-type-specific schema, so all CapSolver task types can be
+used by setting the documented `task.type` and parameters. New CapSolver task types do not require
+a plugin release.
+
+Install it from the repository root:
+
+```bash
+python -m pip install -e . -e examples/plugins/capsolver
+```
+
+Set the API key outside the workflow file:
+
+```bash
+export CAPSOLVER_API_KEY="your-capsolver-api-key"
+```
+
+The complete example is available at [`workflow.yaml`](workflow.yaml). It uses an authorized
+reCAPTCHA v3 test target and writes the raw CapSolver response to `output/capsolver-result.json`.
+
+Enable the plugin in YAML:
+
+```yaml
+version: 1
+
+workflow:
+  name: capsolver-example
+
+plugins:
+  - capsolver
+
+steps:
+  - capsolver_solve:
+      task:
+        type: ImageToTextTask
+        body: BASE64_IMAGE_DATA
+      save_as: solution
+
+  - save_json:
+      path: output/capsolver-result.json
+      data: "{{solution}}"
+```
+
+## Steps
+
+| Step | API operation |
+| --- | --- |
+| `capsolver_create_task` | `createTask` |
+| `capsolver_get_task_result` | `getTaskResult` |
+| `capsolver_solve` | `createTask` plus polling `getTaskResult` |
+| `capsolver_get_token` | `getToken` |
+| `capsolver_get_balance` | `getBalance` |
+| `capsolver_get_state` | CapSolver's documented service-state operation |
+| `capsolver_request` | Any documented CapSolver API endpoint |
+
+`capsolver_solve` handles both synchronous results, such as recognition tasks, and asynchronous
+token tasks. Polling defaults to three seconds and is capped at 120 attempts, matching the
+documented `getTaskResult` limit. Plugin settings can override `base_url`, `app_id`, `timeout`,
+`poll_interval`, and `max_poll_attempts`; step-level `app_id` and `callback_url` take precedence.
+
+The plugin returns CapSolver responses unchanged, including the task-specific `solution` object.
+The generic `task` and `capsolver_request` payloads preserve forward compatibility with all current
+and future task types and API fields. It does not inject solutions into a browser page; browser/page
+integration remains the responsibility of the consuming workflow and target-site authorization.

crawlerflow-0.3.0/examples/plugins/capsolver/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "crawlerflow-capsolver"
+version = "0.1.0"
+description = "CapSolver plugin for Crawlerflow"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "crawlerflow>=0.2.0",
+    "httpx>=0.27",
+    "pydantic>=2.8",
+]
+
+[project.entry-points."crawlerflow.plugins"]
+capsolver = "crawlerflow_capsolver:CapSolverPlugin"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/crawlerflow_capsolver"]