crawler4j-sdk 0.2.0__py3-none-any.whl

crawler4j_sdk/__init__.py

@@ -0,0 +1,58 @@
+"""Crawler4j SDK - 任务脚本开发工具包。
+
+本包聚合导出模块开发所需的稳定契约与基类，供标准模块直接依赖。
+稳定契约类型由 `crawler4j-contracts` 提供，SDK 负责补充 TaskScript、
+TaskFlow 与 ModuleAssembler 等开发入口。
+"""
+
+from crawler4j_sdk._version import get_version
+from crawler4j_sdk.base import TaskScript
+from crawler4j_sdk.workflow import TaskFlow
+from crawler4j_sdk.assembler import ModuleAssembler
+from crawler4j_sdk.env_selector import EnvSelectorInfo, env_selector
+from crawler4j_contracts import (
+    BBox,
+    ClickCaptchaDebugInfo,
+    ClickCaptchaMatchResult,
+    ClickCaptchaOrderedTarget,
+    EnvCandidate,
+    EnvAction,
+    ImageInput,
+    Point,
+    SliderCaptchaDebugInfo,
+    SliderCaptchaMatchResult,
+    TaskContext,
+    TaskResult,
+    TaskSignal,
+    TaskSignalAction,
+    ToolSpec,
+    ToolsCapability,
+)
+
+__version__ = get_version()
+
+# 稳定导出列表（同 MAJOR 版本内冻结）
+__all__ = [
+    # 核心契约类型
+    "TaskScript",
+    "TaskFlow",
+    "ModuleAssembler",
+    "env_selector",
+    "EnvSelectorInfo",
+    "TaskContext",
+    "TaskResult",
+    "TaskSignal",
+    "TaskSignalAction",
+    "EnvAction",
+    "ImageInput",
+    "BBox",
+    "Point",
+    "ToolSpec",
+    "ToolsCapability",
+    "SliderCaptchaDebugInfo",
+    "SliderCaptchaMatchResult",
+    "ClickCaptchaDebugInfo",
+    "ClickCaptchaOrderedTarget",
+    "ClickCaptchaMatchResult",
+    "EnvCandidate",
+]

crawler4j_sdk/_version.py

@@ -0,0 +1,48 @@
+"""Version helpers backed by package metadata and pyproject.toml."""
+
+from __future__ import annotations
+
+import re
+import tomllib
+from importlib import metadata
+from pathlib import Path
+
+
+PACKAGE_NAME = "crawler4j-sdk"
+PACKAGE_ROOT = Path(__file__).resolve().parents[1]
+PYPROJECT_PATH = PACKAGE_ROOT / "pyproject.toml"
+BASE_VERSION_RE = re.compile(r"^v?(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)")
+
+
+def _load_version_from_metadata() -> str | None:
+    try:
+        return metadata.version(PACKAGE_NAME)
+    except metadata.PackageNotFoundError:
+        return None
+
+
+def _load_version_from_pyproject() -> str:
+    with PYPROJECT_PATH.open("rb") as f:
+        pyproject = tomllib.load(f)
+    return str(pyproject["project"]["version"])
+
+
+def get_version() -> str:
+    """Resolve the package version from build metadata or the source pyproject."""
+    return _load_version_from_metadata() or _load_version_from_pyproject()
+
+
+def get_base_version(version: str | None = None) -> str:
+    """Extract major.minor.patch from the declared version string."""
+    resolved = version or get_version()
+    match = BASE_VERSION_RE.match(resolved.strip())
+    if not match:
+        raise ValueError(f"Unsupported version format: {resolved}")
+    return f"{match.group('major')}.{match.group('minor')}.{match.group('patch')}"
+
+
+def get_compatible_dependency_spec() -> str:
+    """Build the default scaffold dependency range for the current SDK version."""
+    base_version = get_base_version()
+    major = int(base_version.split(".", 1)[0])
+    return f"{PACKAGE_NAME}>={base_version},<{major + 1}.0.0"

crawler4j_sdk/assembler.py

@@ -0,0 +1,216 @@
+"""Module Assembler for Crawler4j modules.
+
+This module provides the ModuleAssembler class which handles automatic discovery
+of tasks and workflows within a module, and provides a standardized entry point.
+"""
+
+import importlib
+import inspect
+import logging
+from pathlib import Path
+from pkgutil import iter_modules
+from typing import Any, Callable, Dict, Optional, Type
+
+import yaml
+
+from crawler4j_sdk.base import TaskScript
+from crawler4j_sdk.context import TaskContext
+from crawler4j_sdk.env_selector import EnvSelectorInfo, get_env_selector_info, invoke_env_selector
+from crawler4j_sdk.result import TaskResult
+from crawler4j_sdk.workflow import TaskFlow
+
+logger = logging.getLogger(__name__)
+
+
+class ModuleAssembler:
+    """Handles discovery and execution of module components."""
+
+    def __init__(self, package_root: Path, module_name: str, default_workflow: str = ""):
+        self.package_root = package_root
+        self.module_name = module_name
+        self.default_workflow = default_workflow
+        self.task_scripts: Dict[str, Type[TaskScript]] = {}
+        self.workflows: Dict[str, Type[TaskFlow]] = {}
+        self.discovery_errors: Dict[str, str] = {}
+
+        # Hooks that can be overridden by module_runtime.py
+        self.hooks: Dict[str, Callable] = {}
+        self.env_selectors: Dict[str, Callable] = {}
+        self.env_selector_infos: Dict[str, EnvSelectorInfo] = {}
+
+        self._load_manifest()
+        self._discover()
+        self._load_runtime_extensions()
+
+    def _load_manifest(self):
+        """Load default_workflow from module.yaml if not set."""
+        manifest_path = self.package_root / "module.yaml"
+        if manifest_path.exists():
+            try:
+                with open(manifest_path, "r", encoding="utf-8") as f:
+                    manifest = yaml.safe_load(f)
+                    if manifest and not self.default_workflow:
+                        workflows = manifest.get("workflows", [])
+                        if workflows and isinstance(workflows, list):
+                            self.default_workflow = workflows[0].get("name", "")
+            except Exception as e:
+                logger.warning(f"Failed to load manifest at {manifest_path}: {e}")
+
+    def _load_registry(self, subpackage: str, base_cls: type) -> Dict[str, type]:
+        registry: Dict[str, type] = {}
+        package_dir = self.package_root / subpackage
+        if not package_dir.exists():
+            return registry
+
+        full_package_name = f"{self.module_name}.{subpackage}"
+        for module_info in iter_modules([str(package_dir)]):
+            if module_info.name.startswith("_"):
+                continue
+            import_target = f"{full_package_name}.{module_info.name}"
+            try:
+                module = importlib.import_module(import_target)
+                for attr_name in dir(module):
+                    candidate = getattr(module, attr_name)
+                    if (
+                        inspect.isclass(candidate)
+                        and issubclass(candidate, base_cls)
+                        and candidate is not base_cls
+                        ):
+                        key = getattr(candidate, "name", "") or module_info.name
+                        registry[key] = candidate
+            except Exception as exc:
+                error_message = (
+                    f"Failed to import {import_target} during {subpackage} discovery: "
+                    f"{exc.__class__.__name__}: {exc}"
+                )
+                self.discovery_errors[module_info.name] = error_message
+                logger.exception(error_message)
+                continue
+        return registry
+
+    def _discover(self):
+        """Automatically discover tasks and workflows."""
+        self.task_scripts = self._load_registry("tasks", TaskScript)
+        self.workflows = self._load_registry("workflows", TaskFlow)
+
+    def _load_runtime_extensions(self):
+        """Load optional module_runtime.py extensions."""
+        runtime_path = self.package_root / "module_runtime.py"
+        if runtime_path.exists():
+            try:
+                runtime_module = importlib.import_module(f"{self.module_name}.module_runtime")
+
+                # Load hooks if present
+                hook_names = [
+                    "prepare_env",
+                    "init_env",
+                    "before_run",
+                    "on_success",
+                    "on_failure",
+                    "on_timeout",
+                    "on_cleanup",
+                ]
+                for name in hook_names:
+                    if hasattr(runtime_module, name):
+                        self.hooks[name] = getattr(runtime_module, name)
+
+                # Allow overriding default_workflow
+                if hasattr(runtime_module, "DEFAULT_WORKFLOW"):
+                    self.default_workflow = getattr(runtime_module, "DEFAULT_WORKFLOW")
+
+                # Allow manual registration/override of tasks/workflows
+                if hasattr(runtime_module, "TASK_SCRIPTS"):
+                    self.task_scripts.update(getattr(runtime_module, "TASK_SCRIPTS"))
+                if hasattr(runtime_module, "WORKFLOWS"):
+                    self.workflows.update(getattr(runtime_module, "WORKFLOWS"))
+
+                for attr_name in dir(runtime_module):
+                    candidate = getattr(runtime_module, attr_name)
+                    info = get_env_selector_info(candidate)
+                    if not info:
+                        continue
+                    self.env_selectors[info.name] = candidate
+                    self.env_selector_infos[info.name] = info
+
+            except Exception as e:
+                logger.exception(f"Failed to load runtime extensions: {e}")
+
+    @staticmethod
+    def _normalize_result_payload(result: object) -> TaskResult:
+        if isinstance(result, TaskResult):
+            return result
+        if result is None:
+            return TaskResult.ok()
+        if isinstance(result, dict):
+            return TaskResult.ok(data=result)
+        return TaskResult.ok(data={"value": result})
+
+    async def _run_task_script(self, script_cls: Type[TaskScript], ctx: TaskContext) -> TaskResult:
+        script = script_cls()
+        result = await script.execute(ctx)
+        return self._normalize_result_payload(result)
+
+    async def _run_task_flow(self, flow_cls: Type[TaskFlow], ctx: TaskContext) -> TaskResult:
+        flow = flow_cls()
+        result = await flow.run(ctx)
+        if result is None:
+            return TaskResult.ok(data=dict(ctx.state))
+        return self._normalize_result_payload(result)
+
+    async def _subtask_executor(self, task_name: str, ctx: TaskContext) -> TaskResult:
+        if task_name not in self.task_scripts:
+            raise ValueError(f"Unknown subtask: {task_name}")
+        return await self._run_task_script(self.task_scripts[task_name], ctx)
+
+    @staticmethod
+    def _resolve_workflow_name(context: TaskContext, default_workflow: str) -> str:
+        """Resolve workflow selection from runtime only."""
+        runtime = getattr(context, "runtime", None)
+        if isinstance(runtime, dict):
+            workflow_name = runtime.get("workflow")
+            if isinstance(workflow_name, str) and workflow_name.strip():
+                return workflow_name
+        return default_workflow
+
+    async def run(self, context: TaskContext) -> TaskResult:
+        """Module execution entry point."""
+        workflow_name = self._resolve_workflow_name(context, self.default_workflow)
+
+        if workflow_name in self.workflows:
+            context._subtask_executor = self._subtask_executor
+            return await self._run_task_flow(self.workflows[workflow_name], context)
+
+        if workflow_name in self.task_scripts:
+            return await self._run_task_script(self.task_scripts[workflow_name], context)
+
+        discovery_error = self.discovery_errors.get(workflow_name)
+        if discovery_error:
+            raise ValueError(f"Workflow or task not found: {workflow_name}. {discovery_error}")
+
+        raise ValueError(f"Workflow or task not found: {workflow_name}")
+
+    def get_hook(self, name: str) -> Optional[Callable]:
+        """Get a registered hook by name."""
+        return self.hooks.get(name)
+
+    def get_env_selector(self, name: str) -> Optional[Callable]:
+        """Get a registered env selector callback by name."""
+        return self.env_selectors.get(name)
+
+    def list_env_selectors(self) -> list[EnvSelectorInfo]:
+        """List registered env selectors in stable name order."""
+        return [
+            self.env_selector_infos[name]
+            for name in sorted(self.env_selector_infos)
+        ]
+
+    async def run_env_selector(
+        self,
+        name: str,
+        context: TaskContext,
+        candidates: list[Any],
+    ) -> Any:
+        selector = self.get_env_selector(name)
+        if not selector:
+            raise ValueError(f"Env selector not found: {name}")
+        return await invoke_env_selector(selector, context, candidates)

crawler4j_sdk/base.py

@@ -0,0 +1,90 @@
+"""TaskScript 基类定义。
+
+本模块定义了 Crawler4j SDK 的核心契约之一：TaskScript（原子任务基类）。
+所有任务脚本必须继承此类并实现 execute 方法。
+
+稳定契约 (Stable API - 同 MAJOR 版本内冻结):
+    - 类属性: name, display_name, description, default_config
+    - 方法: execute
+
+参考规格: docs/02-requirements/reference-srs/06-sdk/06-1-taskscript.md
+"""
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from crawler4j_sdk.context import TaskContext
+    from crawler4j_sdk.result import TaskResult
+
+
+class TaskScript(ABC):
+    """原子任务基类。
+    
+    TaskScript 是最小可执行单元：给定上下文与配置，执行一次业务动作并返回结构化结果。
+    
+    类属性 (Stable):
+        name: 任务唯一标识符 (MUST)，用于调度、引用和追溯。
+        display_name: 面向 UI/日志的可读名称 (SHOULD)。
+        description: 简要说明 (SHOULD)。
+        default_config: 默认配置字典 (MAY)，运行时会与外部配置深合并。
+    
+    并发语义:
+        - 单个 TaskScript 实例不要求线程安全
+        - 框架不得并发调用同一实例的 execute
+        - 每次执行应使用新实例，避免隐式共享状态
+    
+    示例:
+        >>> from crawler4j_sdk import TaskScript, TaskContext, TaskResult
+        >>>
+        >>> class LoginTask(TaskScript):
+        ...     name = "login"
+        ...     display_name = "用户登录"
+        ...     description = "执行网站登录流程"
+        ...     default_config = {"timeout": 30}
+        ...
+        ...     async def execute(self, ctx: TaskContext) -> TaskResult:
+        ...         await ctx.page.goto("https://example.com/login")
+        ...         # ... 登录逻辑
+        ...         return TaskResult.ok(message="登录成功")
+    """
+    
+    # === 类属性 (Stable) ===
+    
+    name: str = ""
+    """任务唯一标识符 (MUST)。用于调度、引用和追溯，应保持稳定。"""
+    
+    display_name: str = ""
+    """面向 UI/日志的可读名称 (SHOULD)。"""
+    
+    description: str = ""
+    """任务简要说明 (SHOULD)。"""
+    
+    default_config: dict[str, Any] = {}
+    """默认配置字典 (MAY)。运行时会与外部配置深合并，外部配置优先。"""
+    
+    # === 主执行方法 (Stable) ===
+    
+    @abstractmethod
+    async def execute(self, ctx: "TaskContext") -> "TaskResult":
+        """执行任务的主方法 (MUST 实现)。
+        
+        这是 TaskScript 的核心方法，子类必须实现。
+        方法应完成一次完整的业务动作并返回结构化结果。
+        
+        Args:
+            ctx: 任务执行上下文，提供 page、config、runtime、logger、tools 等能力。
+                 脚本应只通过 ctx 获取框架能力，不得直接耦合 Core 内部对象。
+        
+        Returns:
+            TaskResult: 执行结果。成功使用 TaskResult.ok()，失败使用 TaskResult.fail()。
+
+        Raises:
+            Exception: 允许抛出异常，由宿主 ATM 统一处理并映射到失败生命周期。
+
+        Note:
+            - 业务失败（可预期）建议返回 TaskResult.fail()
+            - 不可预期异常可直接抛出，由运行时处理
+            - 应避免长时间阻塞调用，使用 async I/O
+        """
+        raise NotImplementedError

crawler4j_sdk/cli/__init__.py

@@ -0,0 +1,13 @@
+"""CLI 命令模块。"""
+
+from __future__ import annotations
+
+
+def main() -> int:
+    """惰性导入 CLI 入口，避免 `python -m ...commands` 时重复预加载。"""
+    from crawler4j_sdk.cli.commands import main as commands_main
+
+    return commands_main()
+
+
+__all__ = ["main"]