recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/connectors/connectors/clickhouse.py

@@ -0,0 +1,219 @@
+from typing import Any
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.datasource import DataSourceWrapper
+from recurvedata.connectors.dbapi import DBAPIBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "clickhouse"
+UI_CONNECTION_TYPE = "ClickHouse"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class ClickhouseConnector(DBAPIBase):
+    SYSTEM_DATABASES = [
+        "system",
+        "information_schema",
+    ]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    setup_extras_require = ["clickhouse-sqlalchemy"]
+    driver = "clickhouse+native"
+    group = [ConnectorGroup.DESTINATION]
+
+    # Official ClickHouse data types from documentation
+    available_column_types = [
+        # Base types from DBAPIBase
+        *DBAPIBase.available_column_types,
+        # Integer types (official)
+        "int8",
+        "int16",
+        "int32",
+        "int64",
+        "int128",
+        "int256",
+        "uint8",
+        "uint16",
+        "uint32",
+        "uint64",
+        "uint128",
+        "uint256",
+        # Float types (official)
+        "float32",
+        "float64",
+        "bfloat16",
+        # String types (official)
+        "string",
+        "fixedstring",
+        # Date/Time types (official)
+        "date",
+        "date32",
+        "datetime",
+        "datetime64",
+        # Boolean type (official)
+        "boolean",
+        # UUID type (official)
+        "uuid",
+        # Network types (official)
+        "ipv4",
+        "ipv6",
+        # Complex types (official)
+        "array",
+        "tuple",
+        "map",
+        "variant",
+        # Enum types (official)
+        "enum",
+        "enum8",
+        "enum16",
+        # Wrapper types (official)
+        "nullable",
+        "lowcardinality",
+        # Aggregate function types (official)
+        "aggregatefunction",
+        "simpleaggregatefunction",
+        # Special types (official)
+        "nested",
+        "dynamic",
+        "json",
+        # Decimal type (official)
+        "decimal",
+        "decimal32",
+        "decimal64",
+        "decimal128",
+        "decimal256",
+    ]
+
+    column_type_mapping = {
+        "integer": [
+            "Int8",
+            "Int16",
+            "Int32",
+            "Int64",
+            "Int128",
+            "Int256",
+            "UInt8",
+            "UInt16",
+            "UInt32",
+            "UInt64",
+            "UInt128",
+            "UInt256",
+        ],
+        "string": ["String", "FixedString", "LowCardinality(String)"],
+    }
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+                "description": _l("The name of the database to connect to"),
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "description": _l(
+                    "Native Protocol port number for the ClickHouse server connection. "
+                    "Normally, default is 9000 for native protocol, use 9440 if secure SSL/TLS is enabled."
+                ),
+                "default": 9000,
+            },
+            "secure": {
+                "type": "boolean",
+                "title": _l("Use Secure Connection"),
+                "description": _l("Enable secure SSL/TLS connection to ClickHouse"),
+                "default": False,
+            },
+        },
+        "order": ["host", "port", "user", "password", "database", "secure"],
+        "required": [
+            "host",
+        ],
+        "secret": ["password"],
+    }
+
+    def _extract_column_name(self, column_type):
+        if column_type.__visit_name__ == "nullable":
+            return column_type.nested_type.__visit_name__
+        return column_type.__visit_name__
+
+    def sqlalchemy_column_type_code_to_name(self, type_code: Any, size: int | None = None) -> str:
+        """
+        Convert SQLAlchemy cursor.description type_code to ClickHouse data type name.
+
+        ClickHouse Connect uses string type names directly in cursor.description,
+        so we can return the type_code as-is if it's a valid ClickHouse type,
+        otherwise fall back to a default type.
+        """
+        if isinstance(type_code, str):
+            # ClickHouse Connect returns string type names directly
+            type_name = type_code.lower()
+
+            # Check if it's a valid ClickHouse type
+            if type_name in self.available_column_types:
+                return type_name
+
+            # Handle common type mappings for ClickHouse
+            type_mapping = {
+                # Common SQL type aliases to ClickHouse types
+                "int": "int32",
+                "integer": "int32",
+                "bigint": "int64",
+                "smallint": "int16",
+                "tinyint": "int8",
+                "double": "float64",
+                "float": "float32",
+                "real": "float32",
+                "varchar": "string",
+                "char": "string",
+                "text": "string",
+                "blob": "string",
+                "timestamp": "datetime",
+                "bool": "boolean",
+                # ClickHouse specific mappings for edge cases
+                "nothing": "string",  # Fallback for Nothing type
+                "expression": "string",  # Fallback for Expression type
+                "set": "string",  # Fallback for Set type
+                "array(string)": "array",
+                "array(date)": "array",
+                "array(datetime)": "array",
+                "array(float64)": "array",
+                "array(float32)": "array",
+                "array(int64)": "array",
+                "array(int32)": "array",
+            }
+
+            mapped_type = type_mapping.get(type_name)
+            if mapped_type and mapped_type in self.available_column_types:
+                return mapped_type
+
+        # For any other type_code format, try to convert to string and check again
+        try:
+            type_str = str(type_code).lower()
+            if type_str in self.available_column_types:
+                return type_str
+        except (ValueError, TypeError):
+            pass
+
+        # Default fallback
+        return "string"
+
+    def convert_config_to_cube_config(
+        self, database: str, schema: str = None, datasource: DataSourceWrapper = None
+    ) -> dict:
+        config = {
+            "type": "clickhouse",
+            "host": self.host,
+            "user": datasource.user,
+            "password": datasource.password,
+            "database": database or self.database,
+        }
+        return config

recurvedata/connectors/connectors/dingtalk.py

@@ -0,0 +1,61 @@
+import requests
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "dingtalk"
+UI_CONNECTION_TYPE = "Ding Talk"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class DingTalk(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.INTEGRATION]
+    enabled = False
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "app_key": {"type": "string", "title": _l("DingTalk App Key")},
+            "app_secret": {"type": "string", "title": _l("DingTalk App Secret")},
+            "phone": {
+                "type": "string",
+                "title": _l("Phone Number"),
+                "description": _l("Phone number associated with the DingTalk account"),
+            },
+        },
+        "order": ["app_key", "app_secret", "phone"],
+        "required": ["app_key", "app_secret"],
+        "secret": ["app_secret"],
+    }
+
+    def test_connection(self):
+        access_token = self.get_token()
+        if self.phone:
+            self.get_user_id_by_phone(access_token)
+
+    def get_token(self):
+        """
+        获取钉钉的访问令牌
+        """
+        url = f"https://oapi.dingtalk.com/gettoken?appkey={self.app_key}&appsecret={self.app_secret}"
+        resp = requests.get(url)
+        data = resp.json()
+        if data.get("errcode"):
+            raise ValueError(f"get_token fail, response data: {data}")
+        return data["access_token"]
+
+    def get_user_id_by_phone(self, access_token):
+        """
+        获取 user id
+        """
+        url = "https://oapi.dingtalk.com/topapi/v2/user/getbymobile"
+        data = {"access_token": access_token, "mobile": self.phone, "support_exclusive_account_search": "true"}
+        resp = requests.post(url=url, data=data)
+        data = resp.json()
+        if data.get("errcode"):
+            raise ValueError(f"get_user_id fail, response data: {data}, please check your phone number")
+        return data.get("result").get("userid")

recurvedata/connectors/connectors/doris.py

@@ -0,0 +1,215 @@
+from functools import cached_property
+from typing import Any
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.const import ENV_VAR_DBT_PASSWORD, ENV_VAR_DBT_USER, SSH_TUNNEL_CONFIG_SCHEMA
+from recurvedata.connectors.datasource import DataSourceWrapper
+from recurvedata.connectors.dbapi import DBAPIBase, with_ssh_tunnel
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "doris"
+UI_CONNECTION_TYPE = "SelectDB(Doris)"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class ApacheDorisConnector(DBAPIBase):
+    SYSTEM_DATABASES = [
+        "information_schema",
+        "mysql",
+        "__internal_schema",
+    ]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    setup_extras_require = ["sqlalchemy-doris[pymysql]"]
+    driver = "doris+pymysql"
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+                "description": _l("The name of the database to connect to"),
+            },
+            "port": {
+                "type": "number",
+                "title": _l("SelectDB(Doris) Port Number"),
+                "description": _l("The port number for connecting to SelectDB(Doris) server on FE"),
+                "default": 9030,
+            },
+            "http_port": {
+                "type": "number",
+                "title": _l("FE HTTP Port"),
+                "description": _l("The HTTP port number for the Doris Frontend (FE) service"),
+                "default": 8030,
+            },
+            "ssh_tunnel": SSH_TUNNEL_CONFIG_SCHEMA,
+        },
+        "order": ["host", "port", "http_port", "user", "password", "database", "ssh_tunnel"],
+        "required": ["host", "http_port"],
+        "secret": ["password"],
+    }
+
+    available_column_types = DBAPIBase.available_column_types + [
+        # Numeric types
+        "tinyint",
+        "integer",  # alias for int
+        "largeint",  # 128-bit signed integer
+        "numeric",  # alias for decimal
+        "real",  # alias for float
+        "double precision",  # alias for double
+        "bigint unsigned",
+        # String types
+        "text",  # alias for string
+        "string",  # alias for varchar
+        # Date and Time types
+        "datetime",  # alias for timestamp
+        "time",
+        # Complex types
+        "array",
+        "map",
+        "struct",
+        "variant",
+        "json",
+        # Special types
+        "bitmap",
+        "hll",
+        "quantile_state",
+        "agg_state",
+        # Boolean type
+        "boolean",
+        "bool",  # alias for
+        # IP types
+        "ipv4",
+        "ipv6",
+    ]
+
+    column_type_mapping = {
+        "integer": ["tinyint", "largeint"],
+        "string": ["text", "string"],
+    }
+
+    def convert_config_to_dbt_profile(self, database: str, schema: str = None) -> dict:
+        return {
+            "host": self.host,
+            "port": self.port,
+            "username": ENV_VAR_DBT_USER,
+            "password": ENV_VAR_DBT_PASSWORD,
+            "schema": database or self.database,
+            "type": "doris",
+            "threads": 10,
+        }
+
+    @with_ssh_tunnel
+    def get_tables(self, database: str = None) -> list[str]:
+        # todo: update doris inspector in sqlalchemy-doris, which return both views and tables
+        table_views: list[str] = super().get_tables(database)
+        for view in self.get_views(database):
+            if view in table_views:
+                table_views.remove(view)
+        return table_views
+
+    @with_ssh_tunnel
+    def get_columns(self, table: str, database: str = None) -> list:
+        database = database or self.database
+        query = f"""
+                SELECT
+                    column_name,
+                    data_type,
+                    column_default,
+                    is_nullable,
+                    column_comment
+                FROM information_schema.columns
+                WHERE table_schema = '{database}'
+                AND table_name = '{table}'
+                """
+        column_metas = []
+        results = self.fetchall(query)
+        for row in results:
+            column_metas.append(
+                {
+                    "name": row[0],
+                    "type": row[1].lower() if row[1] else "",
+                    "default": row[2],
+                    "nullable": row[3] == "YES",
+                    "comment": row[4],
+                }
+            )
+
+        return column_metas
+
+    @cached_property
+    @with_ssh_tunnel
+    def type_code_mapping(self) -> dict:
+        query = "SHOW DATA TYPES"
+        rv = self.fetchall(query)
+        return {row[0]: row[0] for row in rv}
+
+    def sqlalchemy_column_type_code_to_name(self, type_code: Any, size: int | None = None) -> str:
+        # values refer to https://doris.apache.org/docs/table-design/data-type/
+        # values of this mapping must be in available_column_types, or just default as string
+        from pymysql.constants import FIELD_TYPE
+
+        type_code_mapping = {
+            FIELD_TYPE.TINY: "tinyint",
+            FIELD_TYPE.SHORT: "smallint",
+            FIELD_TYPE.LONG: "int",
+            FIELD_TYPE.FLOAT: "float",
+            FIELD_TYPE.DOUBLE: "double",
+            FIELD_TYPE.DECIMAL: "decimal",
+            FIELD_TYPE.NEWDECIMAL: "decimal",
+            FIELD_TYPE.LONGLONG: "bigint",
+            FIELD_TYPE.INT24: "int",
+            FIELD_TYPE.DATE: "date",
+            FIELD_TYPE.NEWDATE: "date",
+            FIELD_TYPE.DATETIME: "datetime",
+            FIELD_TYPE.STRING: "varchar",
+            FIELD_TYPE.VARCHAR: "varchar",
+            FIELD_TYPE.JSON: "json",
+        }
+        return type_code_mapping.get(type_code, "varchar")
+
+    def convert_config_to_cube_config(
+        self, database: str, schema: str = None, datasource: DataSourceWrapper = None
+    ) -> dict:
+        return {
+            "type": "doris",
+            "host": self.host,
+            "port": self.port,
+            "user": datasource.user,
+            "password": datasource.password,
+            "database": database or self.database,
+        }
+
+    @classmethod
+    def order_sql(cls, sql: str, orders: list[dict[str, str]] = None):
+        base_sql = f"SELECT * FROM ({sql}) AS _recurve_limit_subquery"
+        if orders:
+            order_clauses = [f"{order['field']} {order['order']}" for order in orders]
+            base_sql += " ORDER BY " + ", ".join(order_clauses)
+        return base_sql
+
+    @classmethod
+    def limit_sql(cls, sql: str, limit: int = 100, orders: list[dict[str, str]] | None = None, offset: int = 0) -> str:
+        """
+        the sqlglot will convert `timestamp` to `datetime`,
+        which cause this sql: `cast(field as timestamp) as field` to be error in dbt build but success in preview.
+        args:
+            sql: the sql to be limited
+            limit: the limit of the sql
+            orders: the orders of the sql(list[dict[str, str]]), each dict contains `field` and `order`, used in data service preview
+        """
+        base_sql = cls.order_sql(sql, orders)
+        if offset:
+            return f"{base_sql} LIMIT {offset}, {limit}"
+
+        return f"{base_sql} LIMIT {limit}"

recurvedata/connectors/connectors/es.py

@@ -0,0 +1,62 @@
+from sqlalchemy.engine import URL
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.dbapi import DBAPIBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "elasticsearch"
+UI_CONNECTION_TYPE = "Elasticsearch"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class ElasticSearchConnector(DBAPIBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+
+    setup_extras_require = ["elasticsearch-dbapi"]
+    driver = "elasticsearch"
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 9200,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+        },
+        "order": [
+            "host",
+            "port",
+            "user",
+            "password",
+        ],
+        "required": [
+            "host",
+        ],
+        "secret": ["password"],
+    }
+
+    @property
+    def user(self):  # todo
+        return self.conf.get("user")
+
+    @property
+    def password(self):  # todo
+        return self.conf.get("password")
+
+    @property
+    def sqlalchemy_url(self):
+        return URL(self.driver, self.user, self.password, self.host, self.port)
+
+    @classmethod
+    def get_sql_operator_types(cls):
+        return []

recurvedata/connectors/connectors/feishu.py

@@ -0,0 +1,65 @@
+try:
+    from larksuiteoapi import DOMAIN_FEISHU, LEVEL_DEBUG, Config
+    from larksuiteoapi.service.bot.v3.api import Service as BotV3Service
+except ImportError:
+    pass
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "feishu_bot"
+UI_CONNECTION_TYPE = "Feishu"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class FeiShuBot(RecurveConnectorBase):
+    setup_extras_require = ["larksuite-oapi"]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.INTEGRATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "app_id": {"type": "string", "title": _l("Feishu Bot App ID")},
+            "app_secret": {"type": "string", "title": _l("Feishu Bot App Secret")},
+        },
+        "order": [
+            "app_id",
+            "app_secret",
+        ],
+        "required": [
+            "app_id",
+            "app_secret",
+        ],
+        "secret": [
+            "app_secret",
+        ],
+    }
+
+    @property
+    def app_id(self):
+        return self.conf["app_id"]
+
+    @property
+    def app_secret(self):
+        return self.conf["app_secret"]
+
+    @property
+    def feishu_conf(self):
+        if hasattr(self, "_feishu_conf"):
+            return self._feishu_conf
+        self.init_config()
+        return self._feishu_conf
+
+    def init_config(self):
+        app_settings = Config.new_internal_app_settings(app_id=self.app_id, app_secret=self.app_secret)
+        self._feishu_conf = Config(DOMAIN_FEISHU, app_settings, log_level=LEVEL_DEBUG)
+
+    def test_connection(self):
+        service = BotV3Service(self.feishu_conf)
+        resp = service.bots.get().do()
+        if resp.code != 0:
+            raise ValueError(f"{resp.msg} {resp.error}")

recurvedata/connectors/connectors/ftp.py

@@ -0,0 +1,50 @@
+try:
+    from fsspec.implementations.ftp import FTPFileSystem
+except ImportError:
+    FTPFileSystem = None
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.ftp import FTPMixin
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "ftp"
+UI_CONNECTION_TYPE = "FTP"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class FTP(FTPMixin):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 21,
+            },
+        },
+        "order": [
+            "host",
+            "port",
+            "user",
+            "password",
+        ],
+        "required": ["host", "port"],
+        "secret": ["password"],
+    }
+
+    def init_connection(self, conf) -> FTPFileSystem:
+        con = FTPFileSystem(host=conf["host"], port=conf["port"], user=conf["user"], password=conf["password"])
+        self.connector = con
+        return con