recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.487"

recurvedata/client/__init__.py

@@ -0,0 +1,3 @@
+from .client import Client
+
+__all__ = ["Client"]

recurvedata/client/client.py

@@ -0,0 +1,150 @@
+import json
+import logging
+import time
+from typing import Any, TypeVar, overload
+
+import httpx
+from pydantic import BaseModel
+
+from recurvedata.__version__ import __version__
+from recurvedata.config import AgentConfig
+from recurvedata.exceptions import APIError, MaxRetriesExceededException, UnauthorizedError
+
+logger = logging.getLogger(__name__)
+
+ResponseModelType = TypeVar("ResponseModelType", bound=BaseModel)
+
+
+class Client:
+    _config: AgentConfig
+    _client: httpx.Client
+
+    def __init__(self, config: AgentConfig = None):
+        if not config:
+            config = AgentConfig.load()
+        self.set_config(config)
+
+    def set_config(self, config: AgentConfig):
+        self._config = config
+        self._client = httpx.Client(
+            base_url=config.server_url,
+            timeout=config.request_timeout,
+            headers={"User-Agent": f"RecurveLib/{__version__}"},
+        )
+
+    @overload
+    def request(self, method: str, path: str, response_model_class: None = None, retries: int = 3, **kwargs) -> Any:
+        ...
+
+    @overload
+    def request(
+        self, method: str, path: str, response_model_class: type[ResponseModelType], retries: int = 3, **kwargs
+    ) -> ResponseModelType:
+        ...
+
+    def prepare_header(self, kwargs: dict):
+        headers = kwargs.pop("headers", {})
+        headers["Authorization"] = f"Bearer {self._config.agent_id}:{self._config.token.get_secret_value()}"
+        headers["X-Tenant-Domain"] = self._config.tenant_domain
+        kwargs["headers"] = headers
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        response_model_class: type[ResponseModelType] | None = None,
+        retries: int = 1,
+        **kwargs,
+    ) -> Any:
+        self.prepare_header(kwargs)
+        pre_err: httpx.HTTPStatusError | None = None
+        for attempt in range(retries):
+            try:
+                resp = self._client.request(method, path, **kwargs)
+                resp.raise_for_status()
+                resp_content = resp.json()
+
+                # TODO(yangliang): handle errors more gracefully
+                if "code" in resp_content and resp_content["code"] != "0":
+                    raise APIError(f"API request failed: {resp_content['msg']}\n{resp_content.get('data')}")
+
+                if response_model_class is not None:
+                    if "code" in resp_content:
+                        return response_model_class.model_validate(resp_content["data"])
+                    return response_model_class.model_validate(resp_content)
+                return resp_content.get("data")
+            except httpx.HTTPStatusError as e:
+                pre_err = e
+                logger.error(
+                    f"HTTP error on attempt {attempt + 1} for url '{e.request.url}' :"
+                    f" {e.response.status_code} - {e.response.text}"
+                )
+                if e.response.status_code == 401:
+                    raise UnauthorizedError("Unauthorized, please check your agent_id and token")
+            except httpx.RequestError as e:
+                logger.debug(f"Request error on attempt {attempt + 1} for url '{e.request.url}': {e}")
+
+            if attempt < retries - 1:
+                time.sleep(2**attempt)  # Exponential backoff
+            else:
+                err_msg = str(pre_err) if pre_err else ""
+                raise MaxRetriesExceededException(
+                    f"Failed to complete {method} request to {path} after {retries} attempts, {err_msg}"
+                )
+
+    def request_file(
+        self,
+        method: str,
+        path: str,
+        file_name: str,
+        retries: int = 1,
+        **kwargs,
+    ) -> bool:
+        self.prepare_header(kwargs)
+
+        pre_err: httpx.HTTPStatusError | None = None
+        for attempt in range(retries):
+            try:
+                resp = self._client.request(method, path, **kwargs)
+                resp.raise_for_status()
+                try:
+                    resp_content = resp.json()
+
+                    if "code" in resp_content and resp_content["code"] != "0":
+                        raise APIError(f"API request failed: {resp_content['msg']}\n{resp_content.get('data')}")
+                except (json.JSONDecodeError, TypeError, UnicodeDecodeError):
+                    pass
+
+                if not resp.content:
+                    return False
+
+                with open(file_name, "wb") as f:
+                    f.write(resp.content)
+                return True
+
+                # TODO(yangliang): handle errors more gracefully
+            except httpx.HTTPStatusError as e:
+                logger.debug(
+                    f"HTTP error on attempt {attempt + 1} for url '{e.request.url}' :"
+                    f" {e.response.status_code} - {e.response.text}"
+                )
+                pre_err = e
+                if e.response.status_code == 401:
+                    raise UnauthorizedError("Unauthorized, please check your agent_id and token")
+            except httpx.RequestError as e:
+                logger.debug(f"Request error on attempt {attempt + 1} for url '{e.request.url}': {e}")
+
+            if attempt < retries - 1:
+                time.sleep(2**attempt)  # Exponential backoff
+            else:
+                err_msg = str(pre_err) if pre_err else ""
+                raise MaxRetriesExceededException(
+                    f"Failed to complete {method} request to {path} after {retries} attempts {err_msg}"
+                )
+
+    def close(self):
+        self._client.close()
+
+    @property
+    def base_url(self) -> str:
+        return str(self._client.base_url)

recurvedata/client/server_client.py

@@ -0,0 +1,91 @@
+import logging
+import os
+from typing import TYPE_CHECKING, Any
+
+from recurvedata.client.client import Client, ResponseModelType
+from recurvedata.config import AgentConfig
+from recurvedata.exceptions import APIError
+
+if TYPE_CHECKING:
+    from recurvedata.dbt.schemas import CompileResponseWithError, PreviewResponseWithError
+
+logger = logging.getLogger(__name__)
+
+
+class ServerDbtClient(Client):
+    def __init__(
+        self,
+        server_host: str = os.environ.get("RECURVE__DPSERVER__HOST", "http://0.0.0.0:25103"),
+        request_timeout: int = 60,
+    ):
+        config = AgentConfig.load()
+        config.server_host = server_host
+        config.request_timeout = request_timeout
+        super().__init__(config)
+
+    def prepare_header(self, kwargs: dict):
+        pass
+
+    def compile(
+        self, project_id: int, sql: str, alias: str, force_regenerate_dir: bool = False
+    ) -> "CompileResponseWithError":
+        from recurvedata.dbt.schemas import CompilePayload, CompileResponseWithError
+
+        payload = CompilePayload(
+            project_id=project_id,
+            sql=sql,
+            alias=alias,
+            force_regenerate_dir=force_regenerate_dir,
+        )
+        return self.request(
+            "POST", path="/api/dbt/compile", json=payload.model_dump(), response_model_class=CompileResponseWithError
+        )
+
+    def preview(
+        self,
+        project_id: int,
+        sql: str,
+        alias: str,
+        limit: int,
+        force_regenerate_dir: bool = False,
+        is_compiled: bool = False,
+    ) -> "PreviewResponseWithError":
+        from recurvedata.dbt.schemas import PreviewPayload, PreviewResponseWithError
+
+        payload = PreviewPayload(
+            project_id=project_id,
+            sql=sql,
+            alias=alias,
+            limit=limit,
+            force_regenerate_dir=force_regenerate_dir,
+            is_compiled=is_compiled,
+        )
+        return self.request(
+            "POST", path="/api/dbt/preview", json=payload.model_dump(), response_model_class=PreviewResponseWithError
+        )
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        response_model_class: type[ResponseModelType] | None = None,
+        retries: int = 1,
+        **kwargs,
+    ) -> Any:
+        """
+        compared with super().request, this function has no retry logic,
+        to avoid all exception type is MaxRetriesExceededException
+        """
+        self.prepare_header(kwargs)
+        resp = self._client.request(method, path, **kwargs)
+        resp.raise_for_status()
+        resp_content = resp.json()
+
+        if "code" in resp_content and resp_content["code"] != "0":
+            raise APIError(f"API request failed: {resp_content['msg']}\n{resp_content.get('data')}")
+
+        if response_model_class is not None:
+            if "code" in resp_content:
+                return response_model_class.model_validate(resp_content["data"])
+            return response_model_class.model_validate(resp_content)
+        return resp_content.get("data")

recurvedata/config.py

@@ -0,0 +1,99 @@
+import base64
+import json
+import os
+import uuid
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, get_origin
+
+from pydantic import BaseModel, ConfigDict, Field, SecretStr
+from typing_extensions import Self
+
+if TYPE_CHECKING:
+    from pydantic.fields import FieldInfo
+
+RECURVE_HOME = Path(os.environ.get("RECURVE_HOME", Path.home() / ".recurve"))
+CONFIG_FILE_PATH = RECURVE_HOME / "config.json"
+
+EXECUTOR_RECURVE_HOME = Path(os.environ.get("RECURVE__WORKER__BASE", "/opt/recurve"))
+RECURVE_DBT_HOME = EXECUTOR_RECURVE_HOME / "dbt"
+EXECUTOR_META_PATH = EXECUTOR_RECURVE_HOME / "meta"
+EXECUTOR_WORKER_DATA_PATH = EXECUTOR_RECURVE_HOME / "worker_data"
+SERVER_RESULT_STAGING_PATH = EXECUTOR_RECURVE_HOME / "server" / "staging"
+
+PYENV_HOME = Path(os.environ.get("RECURVE__WORKER__PYENV__BASE", "/opt/pyenv"))
+RECURVE_EXECUTOR_PYENV_NAME = "recurve_executor"
+RECURVE_DBT_PYENV_NAME = "dbt"
+RECURVE_EXECUTOR_CLI_NAME = "recurve_executor"
+RECURVE_EXECUTOR_CLI = PYENV_HOME / "versions" / RECURVE_EXECUTOR_PYENV_NAME / "bin" / RECURVE_EXECUTOR_CLI_NAME
+RECURVE_EXECUTOR_DBT_CLI = PYENV_HOME / "versions" / RECURVE_DBT_PYENV_NAME / "bin" / RECURVE_EXECUTOR_CLI_NAME
+
+REDIS_LOCK_URL: str = os.environ.get("RECURVE__REDIS_LOCK_URL", "redis://localhost:6381/1")
+PY_PACKAGES_PATH: str = "/opt/reorc_packages"
+
+
+class AgentConfig(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+    editable_fields: ClassVar[set[str]] = {
+        "token",
+        "request_timeout",
+    }
+
+    agent_id: uuid.UUID = Field(default_factory=uuid.uuid4, description="The unique identifier of the agent.")
+    server_host: str = Field(..., description="The hostname of the server.")
+    tenant_domain: str = Field(..., description="The domain of the tenant.")
+    token: SecretStr = Field(..., description="The authentication token.")
+    request_timeout: int = Field(5, description="The timeout in seconds for HTTP requests.")
+
+    def is_valid(self) -> bool:
+        return all((self.token.get_secret_value(), self.server_host, self.tenant_domain))
+
+    @property
+    def server_url(self) -> str:
+        if self.server_host.startswith("http"):
+            return self.server_host
+        return f"https://{self.server_host}"
+
+    def set_auth_token(self, encoded_token: str):
+        decoded = base64.urlsafe_b64decode(encoded_token.encode()).split(b"::")
+        self.tenant_domain = decoded[0].decode()
+        self.server_host = decoded[1].decode()
+        self.token = SecretStr(base64.urlsafe_b64encode(decoded[2]).decode())
+
+    def clear_auth_token(self):
+        self.token = SecretStr("")
+
+    @classmethod
+    def load(cls, filename: Path | str | None = None) -> Self:
+        if filename is None:
+            filename = CONFIG_FILE_PATH
+        filename = Path(filename)
+        if not filename.exists():
+            cfg = cls(server_host="", tenant_domain="", token="")
+            cfg.save(filename)
+        with open(filename) as f:
+            content = f.read()
+            return cls.model_validate_json(content)
+
+    def save(self, filename: Path | None = None):
+        if filename is None:
+            filename = CONFIG_FILE_PATH
+
+        filename.parent.mkdir(parents=True, exist_ok=True)
+        with open(filename, "w") as f:
+            content: dict = self.model_dump(mode="json")
+            content["token"] = self.token.get_secret_value()
+            f.write(json.dumps(content, indent=2))
+            f.write("\n")
+
+
+def parse_value(key: str, value: str) -> Any:
+    field_info: "FieldInfo" = AgentConfig.model_fields[key]
+    field_type = get_origin(field_info.annotation) or field_info.annotation
+    if field_type is SecretStr:
+        return SecretStr(value)
+    if field_type is bool:
+        return value.lower() in {"true", "yes", "y", "1"}
+    return field_type(value)
+
+
+CONFIG: AgentConfig = AgentConfig.load()

recurvedata/connectors/__init__.py

@@ -0,0 +1,20 @@
+from recurvedata.connectors.pigeon import get_pigeon_connector
+from recurvedata.connectors.service import (
+    get_all_secret_keywords,
+    get_config_schema,
+    get_connection_category,
+    get_connection_class,
+    get_connection_host,
+    get_connection_type,
+    get_connection_ui_category,
+    get_connection_ui_type,
+    get_datasource_by_config,
+    get_datasource_by_name,
+    get_sqlalchemy_url_by_connection,
+    init_connector,
+    list_column_data_types,
+    list_config_schemas,
+    list_dbapi_types,
+    list_juice_sync_able_dbapi_types,
+    test_connection,
+)

recurvedata/connectors/_register.py

@@ -0,0 +1,46 @@
+from typing import TYPE_CHECKING, Type, Union
+
+from recurvedata.connectors.const import get_module_name
+
+if TYPE_CHECKING:
+    from recurvedata.connectors.base import RecurveConnectorBase
+
+_registry = {}
+
+
+def get_connection_class(connection_type: str, only_enabled=True) -> Type["RecurveConnectorBase"]:
+    if connection_type not in _registry:
+        module_name = get_module_name(connection_type)
+        if module_name:
+            __import__(module_name)
+
+    cls = _registry.get(connection_type)
+    if not only_enabled:
+        return cls
+    if cls and cls.enabled:
+        return cls
+
+
+class register_connector_class(object):  # todo: use meta class
+    def __init__(self, connection_types: Union[str, list[str]]):
+        if isinstance(connection_types, str):
+            connection_types = [
+                connection_types,
+            ]
+        self.connection_types: list = connection_types
+
+    def __call__(self, connector_cls):
+        for name in self.connection_types:
+            _registry[name] = connector_cls
+        self.add_connection_type(connector_cls)
+        self.set_connection_keys(connector_cls)
+        return connector_cls
+
+    def add_connection_type(self, connection_cls):
+        if not connection_cls.connection_type:
+            connection_cls.connection_type = self.connection_types[0]
+
+    def set_connection_keys(self, connection_cls):
+        connection_cls.required_keys = connection_cls.config_schema.get("required", [])
+        connection_cls.connection_keys = connection_cls.config_schema.get("order", [])
+        connection_cls.secret_keys = connection_cls.config_schema.get("secret", [])

recurvedata/connectors/base.py

@@ -0,0 +1,111 @@
+from typing import Any
+
+from recurvedata.consts import ConnectionCategory, ConnectorGroup
+from recurvedata.core.configurable import BaseConfigModel, Configurable
+from recurvedata.utils.log import LoggingMixin
+
+
+class BaseConnector(Configurable, LoggingMixin):
+    """Abstract base class for connectors."""
+
+    def __init__(self, config: BaseConfigModel, **kwargs):
+        self.config = config
+        self.kwargs = kwargs
+
+        self._log_config_logger_name = "recurvedata.connectors"
+        if kwargs.get("logger_name"):
+            self._logger_name = kwargs.get("logger_name")
+
+    def connect(self) -> Any:
+        raise NotImplementedError("connect method not implemented")
+
+    def test_connection(self) -> bool:
+        raise NotImplementedError("test_connection method not implemented")
+
+
+class RecurveConnectorBase(object):
+    category: list[ConnectionCategory] = [
+        ConnectionCategory.OTHERS,
+    ]
+    ui_category = ""
+    setup_extras_require = []
+    connection_type = ""
+    ui_connection_type = ""
+    config_schema = {}
+    enabled = True
+    juice_sync_able = False  # 是否可用于 juice sync 同步
+    group: list[ConnectorGroup] = []
+    test_required = True
+    available_column_types = []
+
+    def __init__(self, conf, *args, **kwargs):
+        self.conf = conf
+        self.args = args
+        self.kwargs = kwargs
+
+    def __getattribute__(self, key):  # todo: use s.attr
+        try:
+            return super().__getattribute__(key)
+        except AttributeError as e:
+            if key in self.conf:
+                return self.conf[key]
+            if key in self.kwargs:
+                return self.kwargs[key]
+            if key in self.connection_keys and key not in self.required_keys:
+                return None
+            raise e
+
+    def test_connection(self):
+        pass
+
+    @classmethod
+    def get_secret_keys(cls):
+        return cls.config_schema.get("secret", [])
+
+    @staticmethod
+    def preprocess_conf(conf: dict) -> dict:
+        """
+        在connector 对象初始化之前，调用这个函数，处理一下 conf，
+        原因：
+        1. 有些 json 数据存到数据库后，是 txt 字段，这里处理下
+        2. 增加一些通用的字段 mapping
+        :return: dict
+        """
+        if "passwd" in conf:
+            conf["password"] = conf["passwd"]
+        return conf
+
+    required_keys = []  # todo: cache property from config_schema
+    connection_keys = []
+
+    def juice_sync_path(self, path: str) -> tuple[str, str]:
+        """
+        Return the paths used in juice sync.
+        The first return value is the path with a secret key,
+        and the second return value is the path without a secret key, intended for display purposes.
+        """
+        if not self.juice_sync_able:
+            raise ValueError(f"{self.connection_type} is not juice sync able")
+        raise NotImplementedError
+
+    @classmethod
+    def format_config_schema(cls):
+        return cls.config_schema
+
+    @classmethod
+    def is_dbt_supported(cls):
+        from recurvedata.connectors.const import DBT_SUPPORTED_TYPES
+
+        return cls.connection_type in DBT_SUPPORTED_TYPES
+
+    def convert_config_to_dbt_profile(self, database: str, schema: str = None) -> dict:
+        raise NotImplementedError("convert_config_to_dbt_profile method not implemented")
+
+    @classmethod
+    def is_cube_supported(cls):
+        from recurvedata.connectors.const import CUBE_SUPPORTED_TYPES
+
+        return cls.connection_type in CUBE_SUPPORTED_TYPES
+
+    def convert_config_to_cube_config(self, database: str = None, schema: str = None, ds = None) -> dict:
+        raise NotImplementedError("convert_config_to_cube method not implemented")