crawlerflow 0.2.0__tar.gz → 0.4.0__tar.gz

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/.gitignore

@@ -9,3 +9,6 @@ dist/
 output/
 site/
 
+
+# graft's local graph cache — regenerable, not committed (run `graft build`).
+/graft/

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/CHANGELOG.md

@@ -5,6 +5,35 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and this project
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2026-09-20
+
+### Added
+
+- `filter_json` step for filtering JSON arrays and objects with declarative conditions and
+  selecting only the fields needed for output.
+
+## [0.3.0] - 2026-09-17
+
+### Added
+
+- Workflow-wide proxy support. `WorkflowContext` carries a `proxy_url` that survives parallel loop
+  forks, and the built-in `http_request`, `resolve_location_url`, `enrich_html_links_http`, and
+  `enrich_json_map_locations` steps route their requests through it.
+- `BrowserAdapter.configure_proxy()` extension hook. Adapters that cannot proxy raise a clear
+  error, keeping the contract explicit.
+- Pydoll proxy support for HTTP, HTTPS, and SOCKS5 URLs, including authenticated proxies through a
+  generated extension. The temporary extension directory is removed when the adapter closes.
+- CapSolver reference plugin that solves captcha tasks and exposes the solution to later steps.
+- Webshare reference plugin that selects proxies and publishes the active proxy to the workflow.
+- Combined CapSolver and Webshare example workflows covering browser and HTTP-only Cloudflare
+  challenge flows.
+- Plugin documentation covering proxy-aware plugins and the new adapter hook.
+
+### Fixed
+
+- Disabled Jekyll processing for the published documentation site so workflow expression examples
+  are no longer parsed as Liquid templates.
+
 ## [0.2.0] - 2026-08-18
 
 ### Changed
@@ -36,5 +65,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 - Plugin API with typed settings, lifecycle hooks, steps, filters, and subscribers.
 - `run`, `validate`, `list-steps`, `list-plugins`, and `doctor` CLI commands.
 
+[0.4.0]: https://github.com/mehmetemineker/crawlerflow/releases/tag/v0.4.0
+[0.3.0]: https://github.com/mehmetemineker/crawlerflow/releases/tag/v0.3.0
 [0.2.0]: https://github.com/mehmetemineker/crawlerflow/releases/tag/v0.2.0
 [0.1.0]: https://github.com/mehmetemineker/crawlerflow/releases/tag/v0.1.0

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: crawlerflow
-Version: 0.2.0
+Version: 0.4.0
 Summary: Declarative YAML workflow engine for browser automation and web scraping
 Project-URL: Homepage, https://github.com/mehmetemineker/crawlerflow
 Project-URL: Repository, https://github.com/mehmetemineker/crawlerflow

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/crawlerflow/__init__.py

@@ -5,5 +5,5 @@ from __future__ import annotations
 from crawlerflow.engine.runner import WorkflowRunner
 
 __all__ = ["WorkflowRunner"]
-__version__ = "0.2.0"
+__version__ = "0.4.0"
 

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/crawlerflow/browser/base.py

@@ -66,6 +66,13 @@ class BrowserAdapter(ABC):
     @abstractmethod
     async def screenshot(self, path: Path) -> Path: ...
 
+    def configure_proxy(self, proxy_url: str) -> None:
+        """Configure a proxy before the browser session starts, if supported."""
+
+        raise RuntimeError(
+            f"Browser adapter {type(self).__name__} does not support proxy configuration"
+        )
+
     async def close(self) -> None:
         """Release browser resources when an adapter owns them."""
 

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/crawlerflow/browser/pydoll.py

@@ -6,11 +6,14 @@ import asyncio
 import json
 import logging
 import math
+import shutil
+import tempfile
 import time
 from collections.abc import Mapping
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from pathlib import Path
 from typing import Any
+from urllib.parse import unquote, urlsplit
 
 from crawlerflow.browser.base import BrowserAdapter, BrowserResponse
 
@@ -40,6 +43,7 @@ class PydollBrowserConfig:
     default_wait_timeout: float = 10
     network_idle_period: float = 0.5
     download_directory: Path | None = None
+    proxy_url: str | None = None
 
 
 class PydollBrowserAdapter(BrowserAdapter):
@@ -61,6 +65,7 @@ class PydollBrowserAdapter(BrowserAdapter):
         self._network_callback_ids: list[int] = []
         self._inflight_requests: set[str] = set()
         self._last_network_activity = time.monotonic()
+        self._proxy_extension_dir: Path | None = None
 
     async def goto(self, url: str) -> None:
         tab = await self._get_tab()
@@ -174,6 +179,18 @@ class PydollBrowserAdapter(BrowserAdapter):
         await tab.take_screenshot(path)
         return path
 
+    def configure_proxy(self, proxy_url: str) -> None:
+        """Set an HTTP(S)/SOCKS5 proxy before Chromium is launched."""
+
+        if self._browser is not None or self._tab is not None:
+            raise PydollAdapterError("Proxy must be configured before the browser starts")
+        parsed = urlsplit(proxy_url)
+        if parsed.scheme not in {"http", "https", "socks5"} or not parsed.hostname:
+            raise PydollAdapterError("Proxy URL must include an HTTP(S) or SOCKS5 host")
+        if parsed.port is None or not 1 <= parsed.port <= 65535:
+            raise PydollAdapterError("Proxy URL must include a valid port")
+        self.config = replace(self.config, proxy_url=proxy_url)
+
     async def close(self) -> None:
         if self._tab is not None:
             for callback_id in self._network_callback_ids:
@@ -191,6 +208,9 @@ class PydollBrowserAdapter(BrowserAdapter):
         self._browser = None
         self._network_tracking_enabled = False
         self._inflight_requests.clear()
+        if self._proxy_extension_dir is not None:
+            shutil.rmtree(self._proxy_extension_dir, ignore_errors=True)
+            self._proxy_extension_dir = None
 
     async def _get_tab(self) -> Any:
         if self._tab is None:
@@ -220,10 +240,51 @@ class PydollBrowserAdapter(BrowserAdapter):
                 options.set_default_download_directory(str(self.config.download_directory))
             for argument in self.config.arguments:
                 options.add_argument(argument)
+            if self.config.proxy_url is not None:
+                self._configure_proxy_options(options, self.config.proxy_url)
 
             self._browser = Chrome(options=options)
         self._tab = await self._browser.start()
 
+    def _configure_proxy_options(self, options: Any, proxy_url: str) -> None:
+        parsed = urlsplit(proxy_url)
+        host = parsed.hostname
+        port = parsed.port
+        if host is None or port is None:
+            raise PydollAdapterError("Proxy URL must include a host and port")
+        options.add_argument(f"--proxy-server={parsed.scheme}://{host}:{port}")
+        if parsed.username is not None or parsed.password is not None:
+            if parsed.username is None or parsed.password is None:
+                raise PydollAdapterError("Proxy URL must include both username and password")
+            self._proxy_extension_dir = self._create_proxy_auth_extension(
+                unquote(parsed.username),
+                unquote(parsed.password),
+            )
+            options.add_argument(f"--load-extension={self._proxy_extension_dir}")
+
+    @staticmethod
+    def _create_proxy_auth_extension(username: str, password: str) -> Path:
+        extension_dir = Path(tempfile.mkdtemp(prefix="crawlerflow-proxy-auth-"))
+        manifest = {
+            "manifest_version": 2,
+            "name": "Crawlerflow proxy authentication",
+            "version": "1.0",
+            "permissions": ["webRequest", "webRequestBlocking", "<all_urls>"],
+            "background": {"scripts": ["background.js"]},
+        }
+        background = (
+            "chrome.webRequest.onAuthRequired.addListener("
+            "function(details) { return {authCredentials: {username: "
+            f"{json.dumps(username)}, password: {json.dumps(password)}}}; }}, "
+            "{urls: ['<all_urls>']}, ['blocking']);"
+        )
+        (extension_dir / "manifest.json").write_text(
+            json.dumps(manifest),
+            encoding="utf-8",
+        )
+        (extension_dir / "background.js").write_text(background, encoding="utf-8")
+        return extension_dir
+
     async def _query(self, selector: str, timeout_seconds: float | None = None) -> Any:
         tab = await self._get_tab()
         timeout = timeout_seconds or self.config.default_wait_timeout

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/crawlerflow/engine/context.py

@@ -22,6 +22,7 @@ class WorkflowContext:
     workflow_name: str
     base_path: Path
     browser: BrowserAdapter | None = None
+    proxy_url: str | None = None
     variables: dict[str, Any] = field(default_factory=dict)
     outputs: dict[str, Any] = field(default_factory=dict)
     cookies: dict[str, str] = field(default_factory=dict)
@@ -51,6 +52,7 @@ class WorkflowContext:
             workflow_name=self.workflow_name,
             base_path=self.base_path,
             browser=self.browser,
+            proxy_url=self.proxy_url,
             variables=dict(self.variables),
             outputs=dict(self.outputs),
             cookies=dict(self.cookies),

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/crawlerflow/steps/utility.py

@@ -19,6 +19,7 @@ from crawlerflow.browser import BrowserResponse
 from crawlerflow.engine.context import WorkflowContext
 from crawlerflow.engine.registry import BaseStep, step
 from crawlerflow.events import EventName
+from crawlerflow.expressions.conditions import ConditionOperator
 
 console = Console()
 
@@ -86,6 +87,137 @@ class SaveJsonStep(BaseStep[SaveJsonConfig]):
         )
 
 
+class JsonFilterRule(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    path: str = Field(min_length=1)
+    operator: ConditionOperator
+    value: Any = None
+
+
+class FilterJsonConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    data: Any
+    items_path: str | None = Field(default=None, min_length=1)
+    select: dict[str, str] | None = None
+    where: list[JsonFilterRule] = Field(default_factory=list)
+    save_as: str = Field(pattern=r"^[A-Za-z_]\w*$")
+
+    @field_validator("items_path")
+    @classmethod
+    def validate_items_path(cls, value: str | None) -> str | None:
+        if value is not None and any(not part.strip() for part in value.split(".")):
+            raise ValueError("filter_json items_path contains an empty segment")
+        return value
+
+    @field_validator("select")
+    @classmethod
+    def validate_select_paths(cls, value: dict[str, str] | None) -> dict[str, str] | None:
+        if value is None:
+            return None
+        for output_path, source_path in value.items():
+            if not output_path.strip() or any(
+                not part.strip() for part in output_path.split(".")
+            ):
+                raise ValueError("filter_json select contains an invalid output path")
+            if not source_path.strip() or any(
+                not part.strip() for part in source_path.split(".")
+            ):
+                raise ValueError("filter_json select contains an invalid source path")
+        return value
+
+
+@step("filter_json")
+class FilterJsonStep(BaseStep[FilterJsonConfig]):
+    """Filter JSON objects and arrays before saving or passing them onward."""
+
+    config_model = FilterJsonConfig
+    _missing = object()
+
+    async def execute(self, context: WorkflowContext) -> Any:
+        filtered_data = deepcopy(self.config.data)
+        if self.config.items_path is None:
+            result = self._filter_root(context, filtered_data)
+        else:
+            target = self._lookup(filtered_data, self.config.items_path)
+            if not isinstance(target, list):
+                raise ValueError(
+                    f"filter_json items_path must point to a list: {self.config.items_path}"
+                )
+            self._set_path(
+                filtered_data,
+                self.config.items_path,
+                self._filter_items(context, target),
+            )
+            result = filtered_data
+
+        context.outputs[self.config.save_as] = result
+        return result
+
+    def _filter_root(self, context: WorkflowContext, data: Any) -> Any:
+        if isinstance(data, list):
+            return self._filter_items(context, data)
+        if isinstance(data, dict):
+            if not self._matches(context, data):
+                return None
+            return self._select(data)
+        raise ValueError("filter_json data must be a JSON object or array")
+
+    def _filter_items(self, context: WorkflowContext, items: list[Any]) -> list[Any]:
+        result = []
+        for item in items:
+            if not isinstance(item, dict):
+                if self.config.where or self.config.select is not None:
+                    raise ValueError("filter_json where/select requires object items")
+                result.append(item)
+                continue
+            if self._matches(context, item):
+                result.append(self._select(item))
+        return result
+
+    def _matches(self, context: WorkflowContext, item: dict[str, Any]) -> bool:
+        for rule in self.config.where:
+            value = self._lookup(item, rule.path)
+            if value is self._missing:
+                return False
+            if not context.condition_engine.evaluate(
+                {"left": value, "operator": rule.operator, "right": rule.value}
+            ):
+                return False
+        return True
+
+    def _select(self, item: dict[str, Any]) -> dict[str, Any]:
+        if self.config.select is None:
+            return item
+        result: dict[str, Any] = {}
+        for output_path, source_path in self.config.select.items():
+            value = self._lookup(item, source_path)
+            if value is not self._missing:
+                self._set_path(result, output_path, deepcopy(value))
+        return result
+
+    @classmethod
+    def _lookup(cls, value: Any, path: str) -> Any:
+        current = value
+        for part in path.split("."):
+            if not isinstance(current, dict) or part not in current:
+                return cls._missing
+            current = current[part]
+        return current
+
+    @classmethod
+    def _set_path(cls, value: dict[str, Any], path: str, replacement: Any) -> None:
+        parts = path.split(".")
+        current = value
+        for part in parts[:-1]:
+            child = current.get(part)
+            if not isinstance(child, dict):
+                raise ValueError(f"filter_json path is not an object: {path}")
+            current = child
+        current[parts[-1]] = replacement
+
+
 class ExtractJavascriptArrayConfig(BaseModel):
     model_config = ConfigDict(extra="forbid")
     content: str
@@ -939,7 +1071,10 @@ class EnrichHtmlLinksHttpStep(BaseStep[EnrichHtmlLinksHttpConfig]):
         cache: dict[str, str | None] = {}
         insertions: dict[int, list[tuple[str, str]]] = {}
         last_finished_at: float | None = None
-        async with httpx.AsyncClient(timeout=self.config.timeout) as client:
+        async with httpx.AsyncClient(
+            timeout=self.config.timeout,
+            proxy=context.proxy_url,
+        ) as client:
             for target_index, url in collector.links:
                 if url not in cache:
                     if last_finished_at is not None and self.config.delay > 0:
@@ -1092,6 +1227,7 @@ class ResolveLocationUrlStep(BaseStep[ResolveLocationUrlConfig]):
                 timeout=self.config.timeout,
                 follow_redirects=True,
                 max_redirects=self.config.max_redirects,
+                proxy=context.proxy_url,
             ) as client:
                 response = await client.request(
                     "GET",
@@ -1307,6 +1443,7 @@ class EnrichJsonMapLocationsStep(BaseStep[EnrichJsonMapLocationsConfig]):
             timeout=self.config.timeout,
             follow_redirects=True,
             max_redirects=self.config.max_redirects,
+            proxy=context.proxy_url,
         ) as client:
 
             async def resolve(url: str) -> tuple[str, dict[str, Any] | None]:
@@ -1435,7 +1572,10 @@ class HttpRequestStep(BaseStep[HttpRequestConfig]):
             {"transport": "http", "method": method, "url": self.config.url},
         )
         try:
-            async with httpx.AsyncClient(timeout=self.config.timeout) as client:
+            async with httpx.AsyncClient(
+                timeout=self.config.timeout,
+                proxy=context.proxy_url,
+            ) as client:
                 response = await client.request(
                     method,
                     self.config.url,

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/docs/http-requests.md

@@ -78,6 +78,37 @@ The original `map` field is retained. Successful objects receive numeric `latitu
 field name to also store the resolved URL. `on_url_error` accepts `fail` or `continue`; continued
 errors are available in `json_location_enrichment_errors` workflow storage.
 
+## Filter JSON content
+
+Use `filter_json` to filter an array with one or more `where` rules and optionally project only
+selected fields. `items_path` points to a nested array; without it, the input itself can be an array
+or a single object. Rules are combined with `AND` and use the same operators as the `if` step.
+
+```yaml
+- filter_json:
+    data: "{{response.body}}"
+    items_path: data.items
+    where:
+      - path: status
+        operator: eq
+        value: active
+      - path: score
+        operator: gte
+        value: 50
+    select:
+      id: id
+      name: profile.name
+    save_as: filtered_items
+
+- save_json:
+    path: output/items.json
+    data: "{{filtered_items}}"
+```
+
+`select` maps output field paths to source field paths. Missing selected fields are omitted. Supported
+operators are `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `contains`, `not_contains`, `in`, `not_in`,
+`truthy`, and `falsy`.
+
 ## Extract a JavaScript array
 
 `extract_javascript_array` reads an assigned array from HTML or JavaScript text without starting a

{crawlerflow-0.2.0 → crawlerflow-0.4.0}/docs/plugins.md

@@ -120,6 +120,56 @@ python -m crawlerflow list-plugins
 
 The command displays each plugin name, import target, and owning Python distribution.
 
+## Webshare proxy plugin
+
+The repository also contains a separately installable `crawlerflow-webshare` plugin. It reads the
+Webshare proxy list, selects one valid proxy randomly during workflow startup, and keeps that proxy
+for the workflow's browser and direct HTTP requests:
+
+```bash
+python -m pip install -e . -e examples/plugins/webshare
+export WEBSHARE_API_KEY="your-webshare-api-key"
+```
+
+```yaml
+plugins:
+  - name: webshare
+    settings:
+      mode: direct
+      country_codes: [US]
+      valid_only: true
+      proxy_retry:
+        enabled: true
+        attempts_per_proxy: 3
+        max_proxies: 5
+```
+
+See the complete example in
+[`examples/plugins/webshare/workflow.yaml`](https://github.com/mehmetemineker/crawlerflow/blob/main/examples/plugins/webshare/workflow.yaml).
+
+For an HTTP-only variant without Pydoll, use
+[`cloudflare-challenge-http.yaml`](https://github.com/mehmetemineker/crawlerflow/blob/main/examples/plugins/capsolver-webshare/cloudflare-challenge-http.yaml).
+It fetches the challenge HTML and sends the returned `cf_clearance` cookie with a second direct
+HTTP request through the same proxy.
+
+## Cloudflare Challenge example
+
+The combined CapSolver and Webshare example uses CapSolver's `AntiCloudflareTask`. It opens the
+target with a fixed user agent, passes the same selected Webshare proxy to CapSolver, applies the
+returned clearance cookies, and reloads the target page:
+
+```bash
+python -m pip install -e . -e examples/plugins/capsolver -e examples/plugins/webshare
+export CAPSOLVER_API_KEY="your-capsolver-api-key"
+export WEBSHARE_API_KEY="your-webshare-api-key"
+python -m crawlerflow run examples/plugins/capsolver-webshare/cloudflare-challenge.yaml
+```
+
+Replace the authorized target URL before running. Because the example enables
+`include_credentials: true` to pass the proxy to CapSolver, protect the generated result file;
+it can contain a `cf_clearance` cookie. See
+[`cloudflare-challenge.yaml`](https://github.com/mehmetemineker/crawlerflow/blob/main/examples/plugins/capsolver-webshare/cloudflare-challenge.yaml).
+
 ## Example package
 
 `examples/plugins/example` is a separately installable reference package. It registers a custom

crawlerflow-0.4.0/examples/plugins/capsolver/README.md

@@ -0,0 +1,66 @@
+# Crawlerflow CapSolver plugin
+
+This separately installable plugin exposes CapSolver's generic task API to Crawlerflow. The task
+object is passed through without a task-type-specific schema, so all CapSolver task types can be
+used by setting the documented `task.type` and parameters. New CapSolver task types do not require
+a plugin release.
+
+Install it from the repository root:
+
+```bash
+python -m pip install -e . -e examples/plugins/capsolver
+```
+
+Set the API key outside the workflow file:
+
+```bash
+export CAPSOLVER_API_KEY="your-capsolver-api-key"
+```
+
+The complete example is available at [`workflow.yaml`](workflow.yaml). It uses an authorized
+reCAPTCHA v3 test target and writes the raw CapSolver response to `output/capsolver-result.json`.
+
+Enable the plugin in YAML:
+
+```yaml
+version: 1
+
+workflow:
+  name: capsolver-example
+
+plugins:
+  - capsolver
+
+steps:
+  - capsolver_solve:
+      task:
+        type: ImageToTextTask
+        body: BASE64_IMAGE_DATA
+      save_as: solution
+
+  - save_json:
+      path: output/capsolver-result.json
+      data: "{{solution}}"
+```
+
+## Steps
+
+| Step | API operation |
+| --- | --- |
+| `capsolver_create_task` | `createTask` |
+| `capsolver_get_task_result` | `getTaskResult` |
+| `capsolver_solve` | `createTask` plus polling `getTaskResult` |
+| `capsolver_get_token` | `getToken` |
+| `capsolver_get_balance` | `getBalance` |
+| `capsolver_get_state` | CapSolver's documented service-state operation |
+| `capsolver_request` | Any documented CapSolver API endpoint |
+
+`capsolver_solve` handles both synchronous results, such as recognition tasks, and asynchronous
+token tasks. Polling defaults to three seconds and is capped at 120 attempts, matching the
+documented `getTaskResult` limit. Plugin settings can override `base_url`, `app_id`, `timeout`,
+`poll_interval`, and `max_poll_attempts`; step-level `app_id` and `callback_url` take precedence.
+
+The plugin returns CapSolver responses unchanged, including the task-specific `solution` object.
+The generic `task` and `capsolver_request` payloads preserve forward compatibility with all current
+and future task types and API fields. It does not inject solutions into a browser page; browser/page
+integration remains the responsibility of the consuming workflow and target-site authorization.

crawlerflow-0.4.0/examples/plugins/capsolver/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "crawlerflow-capsolver"
+version = "0.1.0"
+description = "CapSolver plugin for Crawlerflow"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "crawlerflow>=0.2.0",
+    "httpx>=0.27",
+    "pydantic>=2.8",
+]
+
+[project.entry-points."crawlerflow.plugins"]
+capsolver = "crawlerflow_capsolver:CapSolverPlugin"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/crawlerflow_capsolver"]