recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/server/data_service/api.py

@@ -0,0 +1,126 @@
+from fastapi import APIRouter
+
+from recurvedata.core.tracing import Tracing
+from recurvedata.dbt.schemas import PreviewResponseWithError
+from recurvedata.executors.schemas import ResponseModel
+from recurvedata.executors.utils import run_with_result_handling_v2
+from recurvedata.server.data_service.schemas import (
+    DownloadPayload,
+    DownloadResponseWithError,
+    FetchCountPayload,
+    PreviewPayload,
+    PreviewTotalResponseWithError,
+    SqlValidationPayload,
+    SqlValidationResponseWithError,
+)
+from recurvedata.server.data_service.service import DataServiceService
+from recurvedata.utils.sql import trim_replace_special_character
+
+tracer = Tracing()
+router = APIRouter()
+
+
+@router.post("/preview")
+@tracer.create_span(sampling_rate=0.1)
+async def preview(*, payload: PreviewPayload) -> PreviewResponseWithError:
+    sql = trim_replace_special_character(payload.sql, strip_sufix=True)
+
+    service = DataServiceService(
+        project_id=payload.project_id,
+        project_connection_id=payload.project_connection_id,
+    )
+    res: ResponseModel = await run_with_result_handling_v2(
+        service.preview,
+        sql=sql,
+        limit=payload.limit,
+        no_data=payload.no_data,
+        orders=payload.orders,
+    )
+
+    return PreviewResponseWithError.model_validate(res.model_dump())
+
+
+@router.post("/download")
+@tracer.create_span(sampling_rate=0.1)
+async def download(*, payload: DownloadPayload) -> DownloadResponseWithError:
+    sql = trim_replace_special_character(payload.sql, strip_sufix=True)
+
+    service = DataServiceService(
+        project_id=payload.project_id,
+        project_connection_id=payload.project_connection_id,
+    )
+    res: ResponseModel = await run_with_result_handling_v2(
+        service.download,
+        storage_type=payload.storage_type,
+        storage_options=payload.storage_options,
+        sql=sql,
+        orders=payload.orders,
+        fields=payload.fields,
+        file_type=payload.file_type,
+        file_name=payload.file_name,
+        tenant_id=payload.tenant_id,
+        user_id=payload.user_id,
+        project_id=payload.project_id,
+    )
+
+    return DownloadResponseWithError.model_validate(res.model_dump())
+
+
+@router.post("/preview-total")
+@tracer.create_span(sampling_rate=0.1)
+async def preview_total(*, payload: PreviewPayload) -> PreviewTotalResponseWithError:
+    sql = trim_replace_special_character(payload.sql, strip_sufix=True)
+
+    service = DataServiceService(
+        project_id=payload.project_id,
+        project_connection_id=payload.project_connection_id,
+    )
+    res = await run_with_result_handling_v2(
+        service.preview_total,
+        sql=sql,
+        limit=payload.limit,
+        no_data=payload.no_data,
+        orders=payload.orders,
+        offset=payload.offset,
+    )
+
+    return PreviewTotalResponseWithError.model_validate(res.model_dump())
+
+
+@router.post("/fetch-count")
+@tracer.create_span(sampling_rate=0.1)
+async def fetch_count(*, payload: FetchCountPayload) -> dict:
+    sql = trim_replace_special_character(payload.sql, strip_sufix=True)
+
+    service = DataServiceService(
+        project_id=payload.project_id,
+        project_connection_id=payload.project_connection_id,
+    )
+    res: ResponseModel = await run_with_result_handling_v2(
+        service.fetch_count,
+        sql=sql,
+    )
+
+    return res.model_dump()
+
+
+@router.post("/validate-sql")
+@tracer.create_span(sampling_rate=0.1)
+async def validate_sql(*, payload: SqlValidationPayload) -> SqlValidationResponseWithError:
+    """
+    Validate SQL by executing it and checking for syntax/runtime errors.
+    Supports any SQL statement type including DDL, DML, and SELECT.
+    """
+    sql = trim_replace_special_character(payload.sql, strip_sufix=True)
+
+    service = DataServiceService(
+        project_id=payload.project_id,
+        project_connection_id=payload.project_connection_id,
+    )
+    res: ResponseModel = await run_with_result_handling_v2(
+        service.validate_sql,
+        sql=sql,
+        limit=payload.limit,
+    )
+
+    return SqlValidationResponseWithError.model_validate(res.model_dump())

recurvedata/server/data_service/client.py

@@ -0,0 +1,18 @@
+from recurvedata.client.client import Client
+from recurvedata.server.schemas import ConnectionAndVariables
+from recurvedata.utils.helpers import get_env_id
+
+
+class DataServiceClient(Client):
+    def get_connection_and_variables(self, project_id: int, project_connection_id: int) -> ConnectionAndVariables:
+        params = {
+            "env_id": get_env_id(),
+            "project_id": project_id,
+            "project_connection_id": project_connection_id,
+        }
+        return self.request(
+            "GET",
+            path="/api/data-service/connection-and-variable",
+            response_model_class=ConnectionAndVariables,
+            params=params,
+        )

recurvedata/server/data_service/consts.py

@@ -0,0 +1 @@
+FIELD_TYPE_MAP = {"varchar": str, "string": str}

recurvedata/server/data_service/schemas.py

@@ -0,0 +1,68 @@
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+from recurvedata.executors.schemas import Pagination, ResponseModel
+from recurvedata.filestorage import StorageType
+
+
+class PreviewPayload(BaseModel):
+    sql: str
+    project_id: int
+    project_connection_id: int
+    limit: int
+    no_data: bool = False
+    orders: list[dict[str, str]] | None = None
+    offset: int = 0
+
+
+class DownloadPayload(BaseModel):
+    sql: str
+    project_id: int
+    project_connection_id: int
+    orders: list[dict[str, str]] | None = None
+    fields: list[dict] | None = None
+    file_name: str
+    file_type: Literal["csv", "xlsx"]
+    storage_type: StorageType
+    storage_options: dict[str, Any]
+    tenant_id: int
+    user_id: int
+
+
+class DownloadResult(BaseModel):
+    file_name: str
+
+
+class DownloadResponseWithError(ResponseModel):
+    data: DownloadResult | None
+
+
+class PreviewTotalResponseWithError(ResponseModel):
+    data: Pagination[dict[str, Any]] | None
+
+
+class FetchCountPayload(BaseModel):
+    sql: str
+    project_id: int
+    project_connection_id: int
+
+class SqlValidationResult(BaseModel):
+    """Result of SQL validation including any type of SQL statement"""
+    is_valid: bool
+    compiled_sql: str
+    columns: list[dict] = Field(default_factory=list)  # Column info if applicable
+    data: list[list] = Field(default_factory=list)  # Data if applicable and requested
+    error_message: str | None = None
+    error_code: str | None = None
+    error_traceback: str | None = None
+
+class SqlValidationResponseWithError(ResponseModel):
+    data: SqlValidationResult | None
+
+class SqlValidationPayload(BaseModel):
+    sql: str
+    project_id: int
+    project_connection_id: int
+    limit: int = 0  # Default to 0 to avoid large data returns
+    

recurvedata/server/data_service/service.py

@@ -0,0 +1,218 @@
+import os
+import tempfile
+from dataclasses import dataclass
+from functools import cached_property
+from typing import Any, Literal
+
+import pandas as pd
+from loguru import logger
+
+from recurvedata.config import SERVER_RESULT_STAGING_PATH
+from recurvedata.connectors.service import get_datasource_by_config
+from recurvedata.core.templating import Renderer
+from recurvedata.dbt.schemas import PreviewResult
+from recurvedata.exceptions import ERR, WrapRecurveException, wrap_error
+from recurvedata.executors.cli.connector import ConnectionService
+from recurvedata.executors.executor import Executor
+from recurvedata.executors.schemas import Pagination
+from recurvedata.filestorage import StorageType
+from recurvedata.filestorage import factory as filestorage_factory
+from recurvedata.pigeon.dumper import new_to_csv_dumper
+from recurvedata.server.data_service.client import DataServiceClient
+from recurvedata.server.data_service.consts import FIELD_TYPE_MAP
+from recurvedata.server.data_service.schemas import DownloadResult, SqlValidationResult
+from recurvedata.utils.date_time import now
+
+
+@dataclass
+class DataServiceService:
+    project_id: int
+    project_connection_id: int
+    variables: dict = None
+
+    @cached_property
+    def client(self):
+        return DataServiceClient()
+
+    def prepare_variables(self, variables: dict | None) -> dict:
+        logger.info("start process variables")
+        execution_date, schedule_interval = now(), "@daily"
+        processed_variables = Executor.process_variables(variables or {}, {}, execution_date, schedule_interval)
+        result_variables = Renderer().init_context(execution_date, schedule_interval)
+        result_variables.update(processed_variables)
+        return result_variables
+
+    @wrap_error(ERR.DP_FETCH_CONNECTION_FAILED)
+    def fetch_connection_and_variables(self):
+        logger.info("start fetch connection and variables")
+        item = self.client.get_connection_and_variables(self.project_id, self.project_connection_id)
+        con_item = item.connection
+        logger.info("after fetch connection and variables")
+        self.ds = get_datasource_by_config(
+            con_item.type, config=con_item.data, database=con_item.database, schema=con_item.database_schema
+        )
+        self.variables = self.prepare_variables(item.variables)
+
+    def preview(
+        self, sql: str, limit: int, no_data: bool = False, orders: list[dict[str, str]] | None = None, offset: int = 0
+    ) -> PreviewResult:
+        self.fetch_connection_and_variables()
+        rendered_sql = Renderer().render_template(sql, self.variables)
+        if no_data:
+            limit = 0
+
+        con_service = ConnectionService()
+        try:
+            result = con_service.preview_sql(self.ds, rendered_sql, limit, orders=orders, offset=offset)
+            columns = result.columns
+            column_names = set()
+            for col in columns:
+                if col.name in column_names:
+                    raise ValueError(f"duplicate column name: {col.name}, please check your sql query")
+                column_names.add(col.name)
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Failed to preview data: {e}")
+            raise WrapRecurveException(ERR.PREVIEW_DATA_FAILED, e)
+
+    async def download(
+        self,
+        *,
+        storage_type: StorageType,
+        storage_options: dict[str, Any],
+        file_name: str,
+        sql: str,
+        tenant_id: int,
+        project_id: int,
+        user_id: int,
+        orders: list[dict[str, str]] | None = None,
+        file_type: Literal["csv", "xlsx"] = "csv",
+        fields: list[dict] | None = None,
+    ) -> PreviewResult:
+        self.fetch_connection_and_variables()
+        rendered_sql = Renderer().render_template(sql, self.variables)
+
+        connection = self.ds.data.copy()
+        connection.pop("database", None)
+        recurve_con = self.ds.recurve_connector
+        ordered_sql = recurve_con.order_sql(rendered_sql, orders=orders)
+
+        result_file_name = tempfile.mktemp(dir=SERVER_RESULT_STAGING_PATH)
+        os.makedirs(os.path.dirname(result_file_name), exist_ok=True)
+        open(result_file_name, "w").close()
+        logger.info(f"result_file_name: {result_file_name}")
+
+        logger.info("start dump data")
+        dumper = new_to_csv_dumper(
+            dbtype=self.ds.ds_type,
+            connection=connection,
+            database=self.ds.database,
+            connector=self.ds.connector,
+            sql=ordered_sql,
+            filename=result_file_name,
+            write_header=True,
+            merge_files=True,
+        )
+
+        dumper.execute()
+
+        logger.info(f"result_file_name size: {os.path.getsize(result_file_name)}")
+
+        if os.path.getsize(result_file_name) == 0:
+            return DownloadResult(file_name="")
+
+        dtype = {}
+        rename_dict = None
+
+        if fields:
+            for field in fields:
+                field_type = FIELD_TYPE_MAP.get(field.get("field_type"))
+                if field_type:
+                    dtype[field["name"]] = field_type
+
+            _rename_dict = {field["name"]: field["alias"] or field["name"] for field in fields}
+            if any([k != v for k, v in _rename_dict.items()]):
+                rename_dict = _rename_dict
+
+        if file_type == "xlsx" or rename_dict:
+            df = pd.read_csv(result_file_name, dtype=dtype)
+
+            if rename_dict:
+                df.rename(columns=lambda x: rename_dict.get(x, x), inplace=True)
+
+            if file_type == "xlsx":
+                if not result_file_name.endswith(".xlsx"):
+                    result_file_name += ".xlsx"
+                df.to_excel(result_file_name, index=False)
+            else:
+                df.to_csv(result_file_name, index=False)
+
+        if not file_name.endswith(file_type):
+            file_name += f".{file_type}"
+
+        file_name = f"{tenant_id}/{project_id}/{user_id}/{file_name}"
+
+        logger.info("start upload file")
+        storage = filestorage_factory.create(storage_type, storage_options)
+        with open(result_file_name, "rb") as f:
+            file_content = f.read()
+            await storage.write_bytes(file_name, file_content)
+        logger.info("upload file success")
+
+        return DownloadResult(file_name=file_name)
+
+    def preview_total(
+        self,
+        sql: str,
+        limit: int,
+        no_data: bool = False,
+        orders: list[dict[str, str]] | None = None,
+        offset: int = 0,
+    ) -> Pagination[dict[str, Any]]:
+        preview_result = self.preview(sql, limit, no_data, orders, offset)
+
+        items = []
+        for row in preview_result.data:
+            row_data = {}
+            for col, value in zip(preview_result.columns, row):
+                row_data[col.name] = {
+                    "type": col.type,
+                    "name": col.name,
+                    "normalized_type": col.normalized_type,
+                    "value": value,
+                }
+            items.append(row_data)
+
+        con_service = ConnectionService()
+        total = con_service.fetch_total(self.ds, sql)
+
+        return Pagination[dict[str, Any]](
+            items=items,
+            total=total,
+        )
+
+    def fetch_count(self, sql: str) -> int:
+        self.fetch_connection_and_variables()
+        rendered_sql = Renderer().render_template(sql, self.variables)
+        con_service = ConnectionService()
+        return con_service.fetch_total(self.ds, rendered_sql)
+
+    def validate_sql(self, sql: str, limit: int = 0) -> SqlValidationResult:
+        """
+        Validate SQL by executing it and checking for syntax/runtime errors.
+        Supports any SQL statement type including DDL, DML, and SELECT.
+        
+        Args:
+            sql: SQL statement(s) to validate
+            limit: Maximum rows to return (0 = no data returned, just validation)
+            
+        Returns:
+            SqlValidationResult with validation status and error details
+        """
+        
+        self.fetch_connection_and_variables()
+        rendered_sql = Renderer().render_template(sql, self.variables)
+        con_service = ConnectionService()
+        return con_service.validate_sql(self.ds, rendered_sql, limit=limit)

recurvedata/server/dbt/api.py

@@ -0,0 +1,116 @@
+import json
+import tempfile
+
+from fastapi import APIRouter
+from loguru import logger
+
+from recurvedata.config import SERVER_RESULT_STAGING_PATH
+from recurvedata.core.tracing import Tracing
+from recurvedata.dbt.schemas import (
+    BuildPayload,
+    BuildResponseWithError,
+    CompilePayload,
+    CompileResponseWithError,
+    CompileResult,
+    PreviewPayload,
+    PreviewResponseWithError,
+)
+from recurvedata.dbt.service import DbtService
+from recurvedata.dbt.utils import format_var
+from recurvedata.executors.utils import run_with_result_handling
+from recurvedata.utils.sql import trim_replace_special_character
+
+tracer = Tracing()
+router = APIRouter()
+
+
+@router.post("/compile")
+@tracer.create_span(sampling_rate=0.1, context_payload_name="payload")
+def compile(*, payload: CompilePayload) -> CompileResponseWithError:
+    sql = trim_replace_special_character(payload.sql)
+    service = DbtService(
+        project_id=payload.project_id,
+        project_connection_name=payload.alias,
+        force_regenerate_dir=payload.force_regenerate_dir,
+        need_fetch_variable=True,
+    )
+
+    result_file_name = tempfile.mktemp(dir=SERVER_RESULT_STAGING_PATH)
+    logger.info(f"compile result_file_name: {result_file_name}, payload: {payload}")
+
+    run_with_result_handling(
+        service.compile, inline_sql=sql, result_filename=result_file_name, validate_sql=payload.validate_sql
+    )
+    with open(result_file_name, "r") as temp_file:
+        data: dict = json.load(temp_file)
+
+    logger.info(f"finish compile {result_file_name}")
+
+    return CompileResponseWithError.model_validate(data)
+
+
+@router.post("/preview")
+@tracer.create_span(sampling_rate=0.1, context_payload_name="payload")
+async def preview(*, payload: PreviewPayload) -> PreviewResponseWithError:
+    sql = trim_replace_special_character(payload.sql, strip_sufix=True)
+    service = DbtService(
+        project_id=payload.project_id,
+        project_connection_name=payload.alias,
+        force_regenerate_dir=payload.force_regenerate_dir,
+        need_fetch_variable=True,
+    )
+
+    result_file_name = tempfile.mktemp(dir=SERVER_RESULT_STAGING_PATH)
+    logger.info(f"preview result_file_name: {result_file_name}, payload: {payload}")
+
+    run_with_result_handling(
+        service.preview,
+        inline_sql=sql,
+        limit=payload.limit,
+        result_filename=result_file_name,
+        no_data=payload.no_data,
+        is_compiled=payload.is_compiled,
+    )
+    with open(result_file_name, "r") as temp_file:
+        data: dict = json.load(temp_file)
+
+    logger.info(f"finish preview {result_file_name}")
+
+    return PreviewResponseWithError.model_validate(data)
+
+
+@router.post("/build")
+@tracer.create_span(sampling_rate=0.1, context_payload_name="payload")
+def build(*, payload: BuildPayload) -> BuildResponseWithError:
+    service = DbtService(
+        project_id=payload.project_id,
+        project_connection_name=payload.alias,
+        force_regenerate_dir=payload.force_regenerate_dir,
+        need_fetch_variable=True,
+    )
+    service.prepare()
+
+    logger.info(f"build data model {payload.model_name}")
+
+    var_str = format_var(service, service.variables | (payload.variables or {}))
+
+    def run_model(model_name: str, dbt_vars: str = None, full_refresh: bool = False):
+        compiled_code, _ = service._run_model(
+            model_name=model_name,
+            dbt_vars=dbt_vars,
+            full_refresh=full_refresh,
+        )
+        return CompileResult(compiled_sql=compiled_code)
+
+    result_file_name = tempfile.mktemp(dir=SERVER_RESULT_STAGING_PATH)
+    run_with_result_handling(
+        run_model,
+        model_name=payload.model_name,
+        dbt_vars=var_str,
+        full_refresh=payload.full_refresh,
+        result_filename=result_file_name,
+    )
+    with open(result_file_name, "r") as temp_file:
+        data: dict = json.load(temp_file)
+    logger.info(f"finished build model {payload.model_name}")
+    return BuildResponseWithError.model_validate(data)

recurvedata/server/error_code.py

@@ -0,0 +1,49 @@
+from enum import Enum, EnumMeta
+
+
+class ErrorCodeMeta(EnumMeta):
+    _error_codes = set()
+
+    def __new__(metacls, clsname, bases, classdict):
+        enum_members = {k: v for k, v in classdict.items() if not k.startswith("_")}
+        for name, code in enum_members.items():
+            if type(code) is not tuple:
+                continue
+            for error_code in metacls._error_codes:
+                if code[0] == error_code[1][0]:
+                    raise ValueError(f"Error code {code[0]} in {clsname} already exists globally")
+            metacls._error_codes.add((name, code))
+        return super().__new__(metacls, clsname, bases, classdict)
+
+    @classmethod
+    def error_codes(cls):
+        return sorted(list(cls._error_codes), key=lambda x: x[1][0])
+
+
+class BaseErrorCode(Enum, metaclass=ErrorCodeMeta):
+    @property
+    def code(self):
+        return self.value[0]
+
+    @property
+    def message(self):
+        return self.value[1]
+
+    def to_dict(self):
+        return {"code": self.code, "msg": self.message}
+
+    def exception(self, data: dict | None = None, status_code: int | None = None):
+        from recurvedata.server.exceptions import RecurveException
+
+        return RecurveException(self, data, status_code)
+
+
+class ErrorCode(BaseErrorCode):
+    # 00: General
+    INTERNAL_SERVER_ERROR = ("B0001", "Internal Server Error")
+    NOT_IMPLEMENTED = ("B0002", "Not Implemented")
+
+    UNKNOWN_ERROR = ("D0001", "Unknown Error")
+
+
+ERR = ErrorCode  # shortcut

recurvedata/server/exceptions.py

@@ -0,0 +1,19 @@
+from recurvedata.server.error_code import ERR
+
+
+class RecurveException(Exception):
+    _default_code: ERR = ERR.UNKNOWN_ERROR
+    _default_status_code: int = 200
+
+    def __init__(self, code: ERR = None, data: dict | str = None, status_code: int = None):
+        self.code = code or self._default_code
+        self.data = data
+        self.status_code = status_code or self._default_status_code
+
+    def to_dict(self) -> dict:
+        return self.code.to_dict() | {"data": self.data}
+
+
+class InternalServerError(RecurveException):
+    _default_code = ERR.INTERNAL_SERVER_ERROR
+    _default_status_code = 500

recurvedata/server/executor/api.py

@@ -0,0 +1,37 @@
+from fastapi import APIRouter
+from loguru import logger
+
+from recurvedata.core.tracing import Tracing
+from recurvedata.executors.schemas import ResponseModel
+from recurvedata.executors.utils import run_with_result_handling_v2
+from recurvedata.server.executor.schemas import (
+    ValidatePythonScriptPayload,
+    ValidatePythonScriptResponse,
+)
+from recurvedata.server.executor.service import ExecutorService
+
+tracer = Tracing()
+router = APIRouter()
+
+
+@router.post("/validate-python-script")
+@tracer.create_span(sampling_rate=0.1)
+async def validate_python_script(*, payload: ValidatePythonScriptPayload) -> ValidatePythonScriptResponse:
+    """
+    Validate Python script by executing it in the configured Python environment.
+    
+    This endpoint runs the Python code using the same infrastructure as PythonOperator,
+    including proper environment setup and requirements installation.
+    """
+    logger.info(f"validate_python_script: project_id={payload.project_id}, python_env={payload.python_env}")
+
+    res: ResponseModel = await run_with_result_handling_v2(
+        ExecutorService.validate_python_script,
+        payload.timeout,
+        payload.project_id,
+        payload.python_code,
+        payload.python_env,
+    )
+    
+    logger.info("finish validate_python_script")
+    return ValidatePythonScriptResponse.model_validate(res.model_dump())

recurvedata/server/executor/schemas.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from recurvedata.executors.schemas import ResponseModel
+
+
+class ValidatePythonScriptPayload(BaseModel):
+    """Payload for Python script validation request"""
+    
+    project_id: int = Field(..., description="Project ID")
+    python_code: str = Field(..., description="Python code to validate")
+    python_env: str = Field(..., description="Python environment connection name")
+    timeout: int = Field(default=30, description="Validation timeout in seconds")
+
+
+class PythonScriptValidationResult(BaseModel):
+    """Result of Python script validation"""
+    
+    valid: bool = Field(..., description="Whether the script is valid")
+    message: str = Field(default="", description="Validation message")
+    error: Optional[dict] = Field(default=None, description="Error details if validation failed")
+    execution_time_ms: Optional[float] = Field(default=None, description="Execution time in milliseconds")
+    installed_requirements: Optional[str] = Field(default=None, description="Installed requirements")
+
+
+class ValidatePythonScriptResponse(ResponseModel):
+    """Response for Python script validation"""
+    
+    data: Optional[PythonScriptValidationResult] = None