recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/server/executor/service.py

@@ -0,0 +1,220 @@
+import logging
+import os
+import time
+import traceback
+from typing import Dict, Any
+from tempfile import NamedTemporaryFile
+
+from recurvedata.executors.client import ExecutorClient
+from recurvedata.operators.config import CONF
+from recurvedata.config import RECURVE_EXECUTOR_PYENV_NAME, PY_PACKAGES_PATH
+from recurvedata.operators.python_operator.operator import PythonTask, PythonRequirementsMixin
+from recurvedata.server.executor.schemas import PythonScriptValidationResult
+from recurvedata.utils.mp import robust_run_subprocess
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_PY_VERSION = os.environ.get("RECURVE_OPERATOR_PYTHON_DEFAULT_VERSION", "3.11.9")
+
+
+class ExecutorService:
+    """Service for executing Python code validation using the same infrastructure as PythonOperator"""
+
+    @staticmethod
+    def validate_python_script(
+        project_id: int,
+        python_code: str, 
+        python_env_conn_name: str,
+    ) -> PythonScriptValidationResult:
+        """
+        Validate Python script by executing it in the specified Python environment.
+        
+        Uses the same logic as PythonOperator to ensure consistency with actual execution.
+        
+        Args:
+            project_id: Project ID for context
+            python_code: Python code to validate
+            python_env_conn_name: Python environment from project connection name
+            
+        Returns:
+            PythonScriptValidationResult with validation status and details
+        """
+        start_time = time.time()
+        requirements = ""
+        
+        try:
+            logger.info(f"Starting Python script validation for project: {project_id} with python env conn name: {python_env_conn_name}")
+            
+            # Get connection configuration from gateway executor service API by project connection name
+            conn_config = ExecutorService._get_python_env_config(python_env_conn_name, project_id)
+            
+            # Prepare environment using PythonOperator logic (reuse existing methods)
+            pyenv_name: str = conn_config.get("pyenv")
+            py_version: str = conn_config.get("python_version", DEFAULT_PY_VERSION)
+            requirements = conn_config.get("requirements", "")
+            
+            logger.info(f"Preparing Python environment: {pyenv_name} with version {py_version} and requirements: {requirements}")
+            
+            # Reuse existing static methods from PythonTask and PythonRequirementsMixin
+            PythonTask._install_virtualenv(py_version, pyenv_name)
+            if requirements:
+                # Use custom requirements installation to capture error output
+                ExecutorService._install_requirements_with_output_capture(requirements, pyenv_name)
+            
+            # Execute the Python code
+            ExecutorService._execute_python_code(python_code, conn_config, project_id)
+            
+            execution_time_ms = (time.time() - start_time) * 1000
+            
+            logger.info(f"Python script validation completed successfully in {execution_time_ms:.2f}ms")
+            
+            return PythonScriptValidationResult(
+                valid=True,
+                message="Python script executed successfully",
+                execution_time_ms=execution_time_ms,
+                installed_requirements=requirements
+            )
+            
+        except Exception as e:
+            execution_time_ms = (time.time() - start_time) * 1000
+            error_details = {
+                "type": type(e).__name__,
+                "message": str(e),
+                "traceback": traceback.format_exc()
+            }
+            
+            logger.error(f"Python script validation failed: {error_details}")
+            
+            return PythonScriptValidationResult(
+                valid=False,
+                message=f"Python script validation failed: {str(e)}",
+                error=error_details,
+                execution_time_ms=execution_time_ms,
+                installed_requirements=requirements
+            )
+
+    @staticmethod
+    def _get_python_env_config(python_env_project_conn_name: str, project_id: int) -> Dict[str, Any]:
+        """
+        Get Python environment configuration using ExecutorClient.
+        
+        Fetches the real connection configuration from the executor API.
+        """
+        try:
+            logger.info(f"Fetching Python environment configuration for: {python_env_project_conn_name} in project: {project_id}")
+            
+            # Create ExecutorClient to fetch connection configuration
+            executor_client = ExecutorClient()
+            
+            # Use the get_py_conn_configs method to fetch configuration
+            py_conn_config = executor_client.get_py_conn_configs(
+                conn_type="python",
+                pyenv_name=RECURVE_EXECUTOR_PYENV_NAME,
+                project_conn_name=python_env_project_conn_name,
+                project_id=project_id
+            )
+            
+            logger.info(f"Retrieved Python connection config: {py_conn_config}")
+            
+            # Extract the configuration data
+            if py_conn_config and isinstance(py_conn_config, dict):
+                # The response structure should contain the environment configuration
+                requirements_data = py_conn_config.get("requirements", "")
+                
+                # Handle requirements - could be a list or string
+                if isinstance(requirements_data, list):
+                    # Join multiple requirements with newlines
+                    requirements_str = "\n".join(requirements_data)
+                else:
+                    requirements_str = requirements_data or ""
+                
+                config = {
+                    "pyenv": py_conn_config.get("pyenv"),
+                    "python_version": py_conn_config.get("python_version", DEFAULT_PY_VERSION),
+                    "requirements": requirements_str
+                }
+                logger.info(f"Using python env configuration: {config}")
+                return config
+            else:
+                logger.warning(f"No configuration found for {python_env_project_conn_name} in project: {project_id}, using defaults: {DEFAULT_PY_VERSION}")
+                
+        except Exception as e:
+            logger.error(f"Failed to fetch Python environment config for {python_env_project_conn_name} in project: {project_id}: {e}")
+            logger.info("Falling back to default configuration")
+        
+        # Fallback to default configuration
+        return {
+            "pyenv": RECURVE_EXECUTOR_PYENV_NAME,
+            "python_version": DEFAULT_PY_VERSION,
+            "requirements": ""
+        }
+
+    @staticmethod
+    def _install_requirements_with_output_capture(requirements: str, pyenv_name: str) -> None:
+        """
+        Install requirements with output capture to include pip errors in exceptions.
+        
+        This is a modified version of PythonRequirementsMixin._install_requirements
+        that captures the subprocess output for better error reporting.
+        """
+        if pyenv_name != RECURVE_EXECUTOR_PYENV_NAME:
+            requirements += "\nrecurvedata-lib[slim]"
+        if not requirements:
+            return
+            
+        logger.info("installing requirements")
+        
+        # Install recurvedata-lib from local package if it's a new virtualenv
+        if pyenv_name != RECURVE_EXECUTOR_PYENV_NAME:
+            python = CONF.PYENV_PYTHON_PATH.format(pyenv=pyenv_name)
+            output, ret_code = robust_run_subprocess(
+                f"{python} -m pip install -v --no-index --find-links={PY_PACKAGES_PATH} recurvedata-lib[slim]".split(),
+                _logger=logger
+            )
+            if ret_code:
+                raise RuntimeError(f"Failed to install recurvedata-lib:\n{output}")
+        
+        # Install user requirements
+        with NamedTemporaryFile(mode="w+t", prefix="recurve_python_requirements_", suffix=".txt") as requirements_path:
+            requirements_path.write(requirements)
+            requirements_path.flush()
+            python = CONF.PYENV_PYTHON_PATH.format(pyenv=pyenv_name)
+            
+            # Use robust_run_subprocess to capture output
+            output, ret_code = robust_run_subprocess(
+                f"{python} -m pip install -r {requirements_path.name}".split(),
+                _logger=logger
+            )
+            
+            if ret_code:
+                raise RuntimeError(f"Failed to install requirements:\n{output}")
+
+    @staticmethod
+    def _execute_python_code(python_code: str, conn_config: Dict[str, Any], project_id: int) -> None:
+        """
+        Execute Python code using the exact same logic as PythonOperator.
+        
+        This reuses the execution logic from PythonTask.__run_python.
+        """
+        pyenv = conn_config["pyenv"]
+        
+        # Create temporary file for the Python code (same pattern as PythonTask)
+        prefix = f"recurve_python_validation_{project_id}_"
+        with NamedTemporaryFile(mode="w+t", prefix=prefix, suffix=".py") as tmp_file:
+            tmp_file.write(python_code)
+            tmp_file.flush()
+            
+            logger.info(f"Executing Python code in environment {pyenv}")
+            logger.debug(f"Code to execute:\n{python_code}")
+            
+            # Reuse the exact same execution logic as PythonTask.__run_python
+            script_path = os.path.abspath(tmp_file.name)
+            python_path = CONF.PYENV_PYTHON_PATH.format(pyenv=pyenv)
+            os_env = os.environ.copy()
+            
+            output, ret_code = robust_run_subprocess([python_path, script_path], env=os_env, _logger=logger)
+            
+            if ret_code:
+                raise RuntimeError(f"Python Operator Error:\n{output}")  # Same error message as PythonTask
+                
+            logger.info("Python script executed successfully")

recurvedata/server/main.py

@@ -0,0 +1,32 @@
+from fastapi import APIRouter, FastAPI
+from fastapi.responses import ORJSONResponse
+
+from recurvedata.core.tracing import Tracing
+from recurvedata.server.connector.api import router as connector_router
+from recurvedata.server.data_service.api import router as data_service_router
+from recurvedata.server.dbt.api import router as dbt_router
+from recurvedata.server.executor.api import router as executor_router
+from recurvedata.server.schedulers.api import router as schedulers_router
+from recurvedata.utils.log import init_logging, setup_loguru
+
+__all__ = ["create_app"]
+
+
+def create_app() -> FastAPI:
+    init_logging()
+    setup_loguru()
+
+    if not Tracing.is_instantiated():
+        from recurvedata.utils.tracing import create_dp_tracer
+
+        create_dp_tracer("recurve-lib-server")
+
+    app = FastAPI(title="Recurve Lib Server", default_response_class=ORJSONResponse)
+    public_router = APIRouter(prefix="/api")
+    public_router.include_router(dbt_router, prefix="/dbt")
+    public_router.include_router(connector_router, prefix="/connector")
+    public_router.include_router(data_service_router, prefix="/data-service")
+    public_router.include_router(executor_router, prefix="/executor")
+    public_router.include_router(schedulers_router, prefix="/schedulers")
+    app.include_router(public_router)
+    return app

recurvedata/server/schedulers/api.py

@@ -0,0 +1,252 @@
+import json
+import logging
+import subprocess
+from typing import List
+
+from fastapi import APIRouter
+
+from recurvedata.exceptions import RecurveException
+from recurvedata.server.schedulers.schemas import (
+    CreateDagRequest,
+    DeleteDagRequest,
+    RerunJobRunRequest,
+    RerunTaskRunRequest,
+    StartDevRunRequest,
+    TerminateTaskRunRequest,
+    TriggerJobRunRequest,
+)
+from recurvedata.server.schemas import ResponseError, ResponseModel
+from recurvedata.utils.date_time import to_local_datetime
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(tags=["schedulers"])
+job_router = APIRouter()
+task_router = APIRouter()
+
+# ------------------------------
+# job APIs
+# ------------------------------
+
+
+async def _execute_scheduler_command(cmd: List[str], operation_name: str, job_id: int) -> ResponseModel:
+    """
+    Execute a scheduler command and return standardized response.
+
+    Args:
+        cmd: Command to execute
+        operation_name: Name of the operation for logging
+        job_id: Job ID for error context
+    """
+    is_ok = True
+    error = None
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        logger.info(f"{operation_name} {job_id} result: {result.stdout}")
+
+        if result.returncode != 0:
+            logger.error(f"{operation_name} {job_id} failed: {result.stderr}")
+            is_ok = False
+            error = ResponseError.from_recurve_exception(RecurveException(data={"job_id": job_id}))
+
+    except Exception as e:
+        logger.error(f"{operation_name} failed: {e}")
+        is_ok = False
+        error = ResponseError.from_recurve_exception(RecurveException(data={"job_id": job_id}))
+
+    return ResponseModel(ok=is_ok, error=error)
+
+
+@job_router.post("/{job_id}/create")
+async def create_dag(*, job_id: int, body: CreateDagRequest) -> ResponseModel:
+    # First update the DAG
+    update_result = await _execute_scheduler_command(
+        ["recurve_scheduler", "update-dag", "--job_id", str(job_id)], "update dag", job_id
+    )
+
+    if not update_result.ok:
+        return update_result
+
+    # Then activate if requested
+    if body.is_active:
+        activate_result = await _execute_scheduler_command(
+            ["recurve_scheduler", "activate-dag", "--job_id", str(job_id)], "activate dag", job_id
+        )
+        if not activate_result.ok:
+            return activate_result
+
+    logger.info(f"create dag {job_id} success")
+    return ResponseModel(ok=True)
+
+
+@job_router.post("/{job_id}/update")
+async def update_dag(*, job_id: int) -> ResponseModel:
+    return await _execute_scheduler_command(
+        ["recurve_scheduler", "update-dag", "--job_id", str(job_id)], "update dag", job_id
+    )
+
+
+@job_router.post("/{job_id}/activate")
+async def activate_dag(*, job_id: int) -> ResponseModel:
+    return await _execute_scheduler_command(
+        ["recurve_scheduler", "activate-dag", "--job_id", str(job_id)], "activate dag", job_id
+    )
+
+
+@job_router.post("/{job_id}/deactivate")
+async def deactivate_dag(*, job_id: int) -> ResponseModel:
+    return await _execute_scheduler_command(
+        ["recurve_scheduler", "deactivate-dag", "--job_id", str(job_id)], "deactivate dag", job_id
+    )
+
+
+@job_router.post("/{job_id}/delete")
+async def delete_dag(*, job_id: int, body: DeleteDagRequest) -> ResponseModel:
+    return await _execute_scheduler_command(
+        ["recurve_scheduler", "delete-dag", "--job_id", str(job_id), "--job_name", body.job_name], "delete dag", job_id
+    )
+
+
+@job_router.post("/trigger-job-run")
+async def trigger_job_run(
+    *,
+    body: TriggerJobRunRequest,
+) -> ResponseModel:
+    job_id = body.job_id
+    cmd = [
+        "recurve_scheduler",
+        "trigger-job-run",
+        "--job_id",
+        str(job_id),
+        "--execution_date",
+        body.execution_date,
+    ]
+
+    if body.include_past:
+        cmd.append("--include_past")
+    if body.include_future:
+        cmd.append("--include_future")
+    if body.run_type:
+        cmd.extend(["--run_type", body.run_type])
+    if body.conf:
+        cmd.extend(["--conf", json.dumps(body.conf)])
+
+    return await _execute_scheduler_command(cmd, "trigger job run", job_id)
+
+
+@job_router.post("/rerun-job")
+async def rerun_job_run(
+    *,
+    body: RerunJobRunRequest,
+) -> ResponseModel:
+    job_id = body.job_id
+    cmd = ["recurve_scheduler", "rerun-job-run", "--job_id", str(job_id)]
+
+    if body.run_id:
+        cmd.append("--run_id")
+        cmd.append(body.run_id)
+
+    if body.min_execution_date:
+        cmd.append("--min_execution_date")
+        cmd.append(body.min_execution_date)
+    if body.max_execution_date:
+        cmd.append("--max_execution_date")
+        cmd.append(body.max_execution_date)
+    if body.failed_only:
+        cmd.append("--failed_only")
+
+    return await _execute_scheduler_command(cmd, "rerun job run", job_id)
+
+
+@job_router.post("/{job_id}/stop-dev-run")
+async def stop_dev_run(*, job_id: int) -> ResponseModel:
+    logger.info(f"start stop dev run job_id: {job_id}")
+    return await _execute_scheduler_command(
+        ["recurve_scheduler", "stop-dev-run", "--job_id", str(job_id)], "stop dev run", job_id
+    )
+
+
+@job_router.post("/{job_id}/start-dev-run")
+async def start_dev_run(*, job_id: int, body: StartDevRunRequest) -> ResponseModel:
+    logger.info(f"start start dev run job_id: {job_id}")
+    execution_date = to_local_datetime(body.execution_date)
+    return await _execute_scheduler_command(
+        [
+            "recurve_scheduler",
+            "start-dev-run",
+            "--job_id",
+            str(job_id),
+            "--execution_date",
+            execution_date.isoformat(),
+        ],
+        "start dev run",
+        job_id,
+    )
+
+
+# ------------------------------
+# task APIs
+# ------------------------------
+
+
+@task_router.post("/rerun-task")
+async def rerun_task_run(
+    *,
+    body: RerunTaskRunRequest,
+) -> ResponseModel:
+    job_id = body.job_id
+    cmd = [
+        "recurve_scheduler",
+        "rerun-task-run",
+        "--job_id",
+        str(job_id),
+        "--node_key",
+        body.node_key,
+    ]
+
+    if body.run_id:
+        cmd.append("--run_id")
+        cmd.append(body.run_id)
+    if body.min_execution_date:
+        cmd.append("--min_execution_date")
+        cmd.append(body.min_execution_date)
+    if body.max_execution_date:
+        cmd.append("--max_execution_date")
+        cmd.append(body.max_execution_date)
+
+    if body.include_upstream:
+        cmd.append("--include_upstream")
+    if body.include_downstream:
+        cmd.append("--include_downstream")
+    if body.failed_only:
+        cmd.append("--failed_only")
+
+    return await _execute_scheduler_command(cmd, "rerun task run", job_id)
+
+
+@task_router.post("/terminate-task")
+async def terminate_task_run(
+    *,
+    body: TerminateTaskRunRequest,
+) -> ResponseModel:
+    job_id = body.job_id
+    return await _execute_scheduler_command(
+        [
+            "recurve_scheduler",
+            "terminate-task-run",
+            "--job_id",
+            str(job_id),
+            "--run_id",
+            body.run_id,
+            "--node_key",
+            body.node_key,
+        ],
+        "terminate task run",
+        job_id,
+    )
+
+
+# Register sub-routers
+router.include_router(job_router, prefix="/jobs")
+router.include_router(task_router, prefix="/tasks")

recurvedata/server/schedulers/schemas.py

@@ -0,0 +1,50 @@
+from datetime import datetime
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+
+class TriggerJobRunRequest(BaseModel):
+    job_id: int
+    execution_date: str
+    include_past: Optional[bool] = False
+    include_future: Optional[bool] = False
+    run_type: Optional[str] = None
+    conf: Optional[dict[str, Any]] = None
+
+
+class RerunJobRunRequest(BaseModel):
+    job_id: int
+    run_id: str | None = None
+    min_execution_date: str | None = None
+    max_execution_date: str | None = None
+    failed_only: bool = False
+
+
+class RerunTaskRunRequest(BaseModel):
+    job_id: int
+    node_key: str
+    run_id: str | None = None
+    min_execution_date: str | None = None
+    max_execution_date: str | None = None
+    include_upstream: bool = False
+    include_downstream: bool = False
+    failed_only: bool = False
+
+
+class TerminateTaskRunRequest(BaseModel):
+    job_id: int
+    run_id: str
+    node_key: str
+
+
+class DeleteDagRequest(BaseModel):
+    job_name: str
+
+
+class StartDevRunRequest(BaseModel):
+    execution_date: datetime
+
+
+class CreateDagRequest(BaseModel):
+    is_active: bool

recurvedata/server/schemas.py

@@ -0,0 +1,50 @@
+import os
+from traceback import format_exc
+from typing import Any, Self
+
+from pydantic import BaseModel
+
+from recurvedata.exceptions import RecurveException, WrapRecurveException
+from recurvedata.executors.schemas import ConnectionItem
+
+
+class ConnectionAndVariables(BaseModel):
+    connection: ConnectionItem
+    variables: dict
+
+
+class ResponseError(BaseModel):
+    code: str
+    reason: str | None
+    exception: str | None = None
+    traceback: str | None = None
+    data: dict | str | None = None
+
+    @classmethod
+    def from_recurve_exception(cls, recurve_exception: RecurveException) -> Self:
+        if recurve_exception.data:
+            reason = f"{recurve_exception.code.message} {recurve_exception.data}"
+        else:
+            reason = recurve_exception.code.message
+        if isinstance(recurve_exception, WrapRecurveException):
+            exception = str(recurve_exception.exception)
+        else:
+            exception = None
+        return cls(
+            code=recurve_exception.code.code,
+            reason=reason,
+            exception=exception,
+            traceback=format_exc(),
+            data=recurve_exception.data,
+        )
+
+
+class ResponseModel(BaseModel):
+    ok: bool
+    error: ResponseError | None = None
+    data: Any = None
+
+    def model_dump_json_file(self, filename: str):
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        with open(filename, "w") as f:
+            f.write(self.model_dump_json(indent=2))

recurvedata/utils/__init__.py

@@ -0,0 +1,15 @@
+from recurvedata.utils.helpers import extract_dict, first, get_env_id, md5hash, truncate_string, unescape_backslash
+from recurvedata.utils.json import json_dumps, json_loads
+from recurvedata.utils.log import init_logging
+
+__all__ = [
+    "extract_dict",
+    "unescape_backslash",
+    "md5hash",
+    "truncate_string",
+    "get_env_id",
+    "first",
+    "json_dumps",
+    "json_loads",
+    "init_logging",
+]

recurvedata/utils/_typer.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import asyncio
+import functools
+import os
+import traceback
+from typing import Any, Callable
+
+import typer
+from click import ClickException
+from typer.models import CommandFunctionType
+
+RECURVE_TEST_MODE = os.getenv("RECURVE_TEST_MODE", "false").lower() == "true"
+
+
+class RecurveTyper(typer.Typer):
+    """
+    Wrapper for Typer to support async functions and handle errors.
+    """
+
+    def command(self, name: str | None = None, *args, **kwargs) -> Callable[[CommandFunctionType], CommandFunctionType]:
+        def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+            fn = func
+            if asyncio.iscoroutinefunction(func):
+
+                @functools.wraps(func)
+                def sync_wrapper(*inner_args: Any, **inner_kwargs: Any) -> Any:
+                    asyncio.run(func(*inner_args, **inner_kwargs))
+
+                fn = sync_wrapper
+
+            fn = with_cli_exception_handling(fn)
+
+            return super(RecurveTyper, self).command(name=name, *args, **kwargs)(fn)
+
+        return decorator
+
+
+def exit_with_error(message: str, code: int = 1, **kwargs) -> None:
+    """
+    Utility to print a stylized error message and exit with a non-zero code
+    """
+    kwargs.setdefault("fg", typer.colors.RED)
+    typer.secho(message, **kwargs)
+    raise typer.Exit(code)
+
+
+def with_cli_exception_handling(fn):
+    @functools.wraps(fn)
+    def wrapper(*args, **kwargs):
+        try:
+            return fn(*args, **kwargs)
+        except (typer.Exit, typer.Abort, ClickException):
+            raise  # Do not capture click or typer exceptions
+        except Exception:
+            if RECURVE_TEST_MODE:
+                raise  # Reraise exceptions during test mode
+            traceback.print_exc()
+            exit_with_error("An exception occurred.")
+
+    return wrapper