crawlerflow 0.1.0__tar.gz

crawlerflow-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.venv/
+build/
+dist/
+output/
+

crawlerflow-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mehmet Emin Eker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crawlerflow-0.1.0/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.5
+Name: crawlerflow
+Version: 0.1.0
+Summary: Declarative YAML workflow engine for browser automation and web scraping
+Project-URL: Homepage, https://github.com/mehmetemineker/crawlerflow
+Project-URL: Repository, https://github.com/mehmetemineker/crawlerflow
+Project-URL: Issues, https://github.com/mehmetemineker/crawlerflow/issues
+Project-URL: Documentation, https://github.com/mehmetemineker/crawlerflow/tree/main/docs
+Author: Mehmet Emin Eker
+License-Expression: MIT
+License-File: LICENSE
+Keywords: automation,browser,crawler,scraping,workflow,yaml
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP :: Browsers
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.8
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.7
+Requires-Dist: typer>=0.12
+Provides-Extra: browser
+Requires-Dist: pydoll-python<3,>=2.23; extra == 'browser'
+Provides-Extra: dev
+Requires-Dist: pydoll-python<3,>=2.23; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.2; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# CrawlerFlow
+
+CrawlerFlow is a declarative, YAML-based workflow engine for browser automation and web
+scraping. Workflows describe what should happen; adapters and steps decide how it happens.
+
+## Current foundation
+
+- Versioned YAML workflow loading and validation
+- Browser-independent adapter contract and a lazy-starting Pydoll implementation
+- Extensible step registry
+- Isolated plugin API with typed YAML settings, lifecycle hooks, steps, filters, and subscribers
+- Async workflow executor and event bus
+- Variable interpolation and a built-in expression engine
+- Per-run `today` and `now` date variables
+- Nested `foreach`, `foreach_date`, `foreach_select`, and declarative `if` control flow
+- Reusable parameterized workflow macros
+- Per-step retry and continue/fail error policies
+- JSON Lines workflow, step, retry, and request event logging
+- Built-in navigation, interaction, cookies, downloads, screenshots, and selective HTML output
+- `run`, `validate`, `list-steps`, `list-plugins`, and `doctor` CLI commands
+
+## Development
+
+```bash
+python -m pip install -e ".[dev]"
+pytest
+crawlerflow validate examples/basic.yaml
+crawlerflow run examples/basic.yaml
+```
+
+Run multiple workflows sequentially by supplying more paths. Execution stops at the first failed
+workflow:
+
+```bash
+crawlerflow run examples/first-site.yaml examples/second-site.yaml
+```
+
+A directory argument discovers its directly contained `.yaml` and `.yml` files and runs them in
+alphabetical order:
+
+```bash
+crawlerflow run examples
+```
+
+Use asynchronous mode to run every supplied workflow in parallel. All workflows are allowed to
+finish; the command exits with code `1` if any workflow fails:
+
+```bash
+crawlerflow run --mode async examples/first-site.yaml examples/second-site.yaml
+```
+
+Directory discovery can also be combined with parallel execution:
+
+```bash
+crawlerflow run --mode async examples
+```
+
+Add `--progress` to display a live progress bar based on the total workflow count. The bar advances
+as each workflow succeeds or fails in both sequential and asynchronous modes:
+
+```bash
+crawlerflow run --mode async --progress examples
+```
+
+Use `--concurrency` (or `-c`) to limit how many workflows run at the same time in asynchronous
+mode. This avoids starting every HTTP client or browser session simultaneously:
+
+```bash
+crawlerflow run --mode async --concurrency 8 --progress examples
+```
+
+Omitting the option preserves unlimited parallel execution. `--concurrency` accepts positive
+integers and can only be used with `--mode async`.
+
+Install browser support and select Pydoll in a workflow:
+
+```bash
+python -m pip install -e ".[browser,dev]"
+```
+
+```yaml
+browser:
+  engine: pydoll
+  headless: true
+```
+
+Browser-free workflows omit the `browser` section. Applications can also inject another
+`BrowserAdapter` into `WorkflowRunner`. See `docs/http-requests.md` for direct HTTP requests and
+shortened map URL coordinate resolution.
+
+External extensions can register entry points under `crawlerflow.plugins`; workflows activate only
+the plugins they list. See `docs/plugins.md` and the installable `examples/plugins/example` package
+for the plugin contract, discovery command, and packaging example.

crawlerflow-0.1.0/README.md

@@ -0,0 +1,93 @@
+# CrawlerFlow
+
+CrawlerFlow is a declarative, YAML-based workflow engine for browser automation and web
+scraping. Workflows describe what should happen; adapters and steps decide how it happens.
+
+## Current foundation
+
+- Versioned YAML workflow loading and validation
+- Browser-independent adapter contract and a lazy-starting Pydoll implementation
+- Extensible step registry
+- Isolated plugin API with typed YAML settings, lifecycle hooks, steps, filters, and subscribers
+- Async workflow executor and event bus
+- Variable interpolation and a built-in expression engine
+- Per-run `today` and `now` date variables
+- Nested `foreach`, `foreach_date`, `foreach_select`, and declarative `if` control flow
+- Reusable parameterized workflow macros
+- Per-step retry and continue/fail error policies
+- JSON Lines workflow, step, retry, and request event logging
+- Built-in navigation, interaction, cookies, downloads, screenshots, and selective HTML output
+- `run`, `validate`, `list-steps`, `list-plugins`, and `doctor` CLI commands
+
+## Development
+
+```bash
+python -m pip install -e ".[dev]"
+pytest
+crawlerflow validate examples/basic.yaml
+crawlerflow run examples/basic.yaml
+```
+
+Run multiple workflows sequentially by supplying more paths. Execution stops at the first failed
+workflow:
+
+```bash
+crawlerflow run examples/first-site.yaml examples/second-site.yaml
+```
+
+A directory argument discovers its directly contained `.yaml` and `.yml` files and runs them in
+alphabetical order:
+
+```bash
+crawlerflow run examples
+```
+
+Use asynchronous mode to run every supplied workflow in parallel. All workflows are allowed to
+finish; the command exits with code `1` if any workflow fails:
+
+```bash
+crawlerflow run --mode async examples/first-site.yaml examples/second-site.yaml
+```
+
+Directory discovery can also be combined with parallel execution:
+
+```bash
+crawlerflow run --mode async examples
+```
+
+Add `--progress` to display a live progress bar based on the total workflow count. The bar advances
+as each workflow succeeds or fails in both sequential and asynchronous modes:
+
+```bash
+crawlerflow run --mode async --progress examples
+```
+
+Use `--concurrency` (or `-c`) to limit how many workflows run at the same time in asynchronous
+mode. This avoids starting every HTTP client or browser session simultaneously:
+
+```bash
+crawlerflow run --mode async --concurrency 8 --progress examples
+```
+
+Omitting the option preserves unlimited parallel execution. `--concurrency` accepts positive
+integers and can only be used with `--mode async`.
+
+Install browser support and select Pydoll in a workflow:
+
+```bash
+python -m pip install -e ".[browser,dev]"
+```
+
+```yaml
+browser:
+  engine: pydoll
+  headless: true
+```
+
+Browser-free workflows omit the `browser` section. Applications can also inject another
+`BrowserAdapter` into `WorkflowRunner`. See `docs/http-requests.md` for direct HTTP requests and
+shortened map URL coordinate resolution.
+
+External extensions can register entry points under `crawlerflow.plugins`; workflows activate only
+the plugins they list. See `docs/plugins.md` and the installable `examples/plugins/example` package
+for the plugin contract, discovery command, and packaging example.

crawlerflow-0.1.0/crawlerflow/__init__.py

@@ -0,0 +1,9 @@
+"""CrawlerFlow public package."""
+
+from __future__ import annotations
+
+from crawlerflow.engine.runner import WorkflowRunner
+
+__all__ = ["WorkflowRunner"]
+__version__ = "0.1.0"
+

crawlerflow-0.1.0/crawlerflow/__main__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from crawlerflow.cli.app import app
+
+app()
+

crawlerflow-0.1.0/crawlerflow/browser/__init__.py

@@ -0,0 +1,8 @@
+"""Browser adapter contracts and implementations."""
+
+from __future__ import annotations
+
+from crawlerflow.browser.base import BrowserAdapter, BrowserResponse
+from crawlerflow.browser.factory import create_browser_adapter
+
+__all__ = ["BrowserAdapter", "BrowserResponse", "create_browser_adapter"]

crawlerflow-0.1.0/crawlerflow/browser/base.py

@@ -0,0 +1,72 @@
+"""Browser abstraction used by workflow steps."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(slots=True, frozen=True)
+class BrowserResponse:
+    """Serializable response returned by browser-session requests."""
+
+    status_code: int
+    headers: dict[str, str] = field(default_factory=dict)
+    body: str | bytes | None = None
+
+
+class BrowserAdapter(ABC):
+    """Backend-neutral asynchronous browser contract."""
+
+    @abstractmethod
+    async def goto(self, url: str) -> None: ...
+
+    @abstractmethod
+    async def click(self, selector: str) -> None: ...
+
+    @abstractmethod
+    async def fill(self, selector: str, value: str) -> None: ...
+
+    @abstractmethod
+    async def select(self, selector: str, value: str) -> None: ...
+
+    @abstractmethod
+    async def wait(self, selector: str, timeout_seconds: float | None = None) -> None: ...
+
+    @abstractmethod
+    async def wait_network(self, timeout_seconds: float | None = None) -> None: ...
+
+    @abstractmethod
+    async def html(self) -> str: ...
+
+    @abstractmethod
+    async def evaluate(self, script: str) -> Any: ...
+
+    @abstractmethod
+    async def cookies(self) -> dict[str, str]: ...
+
+    @abstractmethod
+    async def set_cookies(self, cookies: dict[str, str]) -> None: ...
+
+    @abstractmethod
+    async def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        headers: dict[str, str] | None = None,
+        data: Any = None,
+    ) -> BrowserResponse: ...
+
+    @abstractmethod
+    async def download(self, url: str, path: Path) -> Path: ...
+
+    @abstractmethod
+    async def screenshot(self, path: Path) -> Path: ...
+
+    async def close(self) -> None:
+        """Release browser resources when an adapter owns them."""
+
+        return None

crawlerflow-0.1.0/crawlerflow/browser/factory.py

@@ -0,0 +1,40 @@
+"""Create browser adapters from workflow configuration."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from crawlerflow.browser.base import BrowserAdapter
+from crawlerflow.workflow.models import BrowserSettings
+
+
+def create_browser_adapter(
+    settings: BrowserSettings,
+    *,
+    base_path: Path,
+) -> BrowserAdapter | None:
+    """Create the configured browser adapter, if browser execution is enabled."""
+
+    if settings.engine is None:
+        return None
+    if settings.engine == "pydoll":
+        from crawlerflow.browser.pydoll import PydollBrowserAdapter, PydollBrowserConfig
+
+        binary_location = settings.binary_location
+        if binary_location is not None and not binary_location.is_absolute():
+            binary_location = base_path / binary_location
+        download_directory = settings.download_directory
+        if download_directory is not None and not download_directory.is_absolute():
+            download_directory = base_path / download_directory
+        return PydollBrowserAdapter(
+            PydollBrowserConfig(
+                headless=settings.headless,
+                binary_location=binary_location,
+                arguments=tuple(settings.arguments),
+                start_timeout=settings.start_timeout,
+                default_wait_timeout=settings.default_wait_timeout,
+                network_idle_period=settings.network_idle_period,
+                download_directory=download_directory,
+            )
+        )
+    raise ValueError(f"Unsupported browser engine: {settings.engine}")