recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/connectors/connectors/slack.py

@@ -0,0 +1,35 @@
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "slack"
+UI_CONNECTION_TYPE = "Slack"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class FeiShuBot(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.INTEGRATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "app_token": {
+                "type": "string",
+                "title": _l("Slack App Token"),
+                "description": _l("The token used to authenticate with Slack API"),
+            },
+        },
+        "required": [
+            "app_token",
+        ],
+        "secret": [
+            "app_token",
+        ],
+    }
+
+    @property
+    def app_token(self):
+        return self.conf["app_token"]

recurvedata/connectors/connectors/spark.py

@@ -0,0 +1,99 @@
+import json
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.consts import ConnectionCategory, ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "spark"
+UI_CONNECTION_TYPE = "Spark"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class SparkSubmit(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    category = [
+        ConnectionCategory.DATABASE,
+    ]
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "submitter": {
+                "type": "string",
+                "title": _l("Spark Submit Command"),
+                "description": _l("Path to spark-submit executable"),
+                "default": "spark-submit",
+            },
+            "env": {
+                "type": "object",
+                "title": _l("Environment Variables"),
+                "description": _l("Environment variables required for Spark execution"),
+                "properties": {
+                    "JAVA_HOME": {"type": "string", "title": "JAVA_HOME"},
+                    "PYSPARK_PYTHON": {"type": "string", "title": "PYSPARK_PYTHON"},
+                    "PYSPARK_DRIVER_PYTHON": {"type": "string", "title": "PYSPARK_DRIVER_PYTHON"},
+                    "HADOOP_USER_NAME": {"type": "string", "title": "HADOOP_USER_NAME"},
+                    "PIGEON_SECRET_KEY": {"type": "string", "title": "PIGEON_SECRET_KEY"},
+                    "PATH": {"type": "string", "title": "PATH"},
+                    "SPARK_CONF_DIR": {"type": "string", "title": "SPARK_CONF_DIR"},
+                    "HADOOP_CONF_DIR": {"type": "string", "title": "HADOOP_CONF_DIR"},
+                    "HIVE_CONF_DIR": {"type": "string", "title": "HIVE_CONF_DIR"},
+                    "HIVE_HOME": {"type": "string", "title": "HIVE_HOME"},
+                },
+                "order": [
+                    "JAVA_HOME",
+                    "PYSPARK_PYTHON",
+                    "PYSPARK_DRIVER_PYTHON",
+                    "HADOOP_USER_NAME",
+                    "PIGEON_SECRET_KEY",
+                    "PATH",
+                    "HIVE_CONF_DIR",
+                    "HADOOP_CONF_DIR",
+                    "SPARK_CONF_DIR",
+                    "HIVE_HOME",
+                ],
+            },
+            "execution_config": {
+                "type": "object",
+                "title": _l("Execution Configuration"),
+                "description": _l("Spark execution settings"),
+                "properties": {
+                    "queue": {
+                        "type": "string",
+                        "title": _l("Queue Name"),
+                        "description": _l("Name of the queue to submit Spark job"),
+                        "default": "recurve",
+                    },
+                    "conf": {
+                        "type": "string",
+                        "title": _l("Additional Configuration"),
+                        "description": _l("Additional Spark configuration in JSON format"),
+                        "ui:options": {"type": "textarea"},
+                    },
+                },
+                "order": ["queue", "conf"],
+            },
+        },
+        "order": ["submitter", "env", "execution_config"],
+        "required": [
+            "submitter",
+        ],
+        "secret": [],
+    }
+
+    def test_connection(self):
+        pass
+
+    @staticmethod
+    def preprocess_conf(data):
+        data = RecurveConnectorBase.preprocess_conf(data)
+        execution_config = data.get("execution_config")
+        if execution_config:
+            execution_config_conf = execution_config.get("conf")
+            if execution_config_conf and isinstance(execution_config_conf, str):
+                execution_config_conf = json.loads(execution_config_conf)
+                execution_config["conf"] = execution_config_conf
+        return data

recurvedata/connectors/connectors/starrocks.py

@@ -0,0 +1,175 @@
+from functools import cached_property
+from typing import Any
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.connectors.mysql import MysqlConnector
+from recurvedata.connectors.const import ENV_VAR_DBT_PASSWORD, ENV_VAR_DBT_USER, SSH_TUNNEL_CONFIG_SCHEMA
+from recurvedata.connectors.datasource import DataSourceWrapper
+from recurvedata.connectors.dbapi import DBAPIBase, with_ssh_tunnel
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "starrocks"
+UI_CONNECTION_TYPE = "StarRocks"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class StarRocksConnector(MysqlConnector):
+    SYSTEM_DATABASES = [
+        "information_schema",
+    ]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+                "description": _l("The name of the database to connect to"),
+            },
+            "port": {
+                "type": "number",
+                "title": _l("MySQL Port Number"),
+                "description": _l("The port number for connecting to MySQL server on FE"),
+                "default": 9030,
+            },
+            "http_port": {
+                "type": "number",
+                "title": _l("FE HTTP Port"),
+                "description": _l("The HTTP port number for the Doris Frontend (FE) service"),
+                "default": 8030,
+            },
+            "ssh_tunnel": SSH_TUNNEL_CONFIG_SCHEMA,
+        },
+        "order": ["host", "port", "http_port", "user", "password", "database", "ssh_tunnel"],
+        "required": ["host", "http_port"],
+        "secret": ["password"],
+    }
+
+    available_column_types = DBAPIBase.available_column_types + [
+        # Numeric types
+        "tinyint",
+        "integer",  # alias for int
+        "largeint",  # 128-bit signed integer
+        "numeric",  # alias for decimal
+        "real",  # alias for float
+        "double precision",  # alias for double
+        # String types
+        "text",  # alias for string
+        "string",  # alias for varchar
+        "binary",
+        "varbinary",
+        # Date and Time types
+        "datetime",  # alias for timestamp
+        "time",
+        # Complex types
+        "array",
+        "map",
+        "struct",
+        "json",
+        # Special types
+        "bitmap",
+        "hll",
+        # Boolean type
+        "boolean",
+        "bool",  # alias for boolean
+    ]
+
+    column_type_mapping = {
+        "integer": ["tinyint", "largeint"],
+        "string": ["text", "string"],
+    }
+
+    def convert_config_to_dbt_profile(self, database: str, schema: str = None) -> dict:
+        return {
+            "host": self.host,
+            "port": self.port,
+            "username": ENV_VAR_DBT_USER,
+            "password": ENV_VAR_DBT_PASSWORD,
+            "schema": database or self.database,
+            "type": "starrocks",  # dbt-starrocks only support python<=3.10
+        }
+
+    def convert_config_to_cube_config(
+        self, database: str, schema: str = None, datasource: DataSourceWrapper = None
+    ) -> dict:
+        return {
+            "type": "mysql",
+            "host": self.host,
+            "port": self.port,
+            "user": datasource.user,
+            "password": datasource.password,
+            "database": database or self.database,
+        }
+
+    @with_ssh_tunnel
+    def get_columns(self, table, database=None):
+        """
+        Override get_columns method to properly handle StarRocks-specific data types
+        """
+        database = database or self.database
+        # Use raw SQL query to get column information instead of relying on SQLAlchemy inspector
+        query = f"""
+        SELECT
+            column_name,
+            data_type,
+            column_default,
+            is_nullable,
+            column_comment
+        FROM information_schema.columns
+        WHERE table_schema = '{database}'
+        AND table_name = '{table}'
+        """
+        column_metas = []
+        results = self.fetchall(query)
+        for row in results:
+            column_metas.append(
+                {
+                    "name": row[0],
+                    "type": row[1].lower() if row[1] else "",
+                    "default": row[2],
+                    "nullable": row[3] == "YES",
+                    "comment": row[4],
+                }
+            )
+
+        return column_metas
+
+    @cached_property
+    @with_ssh_tunnel
+    def type_code_mapping(self) -> dict:
+        pass
+
+    def sqlalchemy_column_type_code_to_name(self, type_code: Any, size: int | None = None) -> str:
+        # values refer to https://docs.starrocks.io/docs/sql-reference/data-types/
+        # values of this mapping must be in available_column_types, or just default as string
+        from pymysql.constants import FIELD_TYPE
+
+        type_code_mapping = {
+            FIELD_TYPE.TINY: "tinyint",
+            FIELD_TYPE.SHORT: "smallint",
+            FIELD_TYPE.LONG: "int",
+            FIELD_TYPE.FLOAT: "float",
+            FIELD_TYPE.DOUBLE: "double",
+            FIELD_TYPE.DECIMAL: "decimal",
+            FIELD_TYPE.NEWDECIMAL: "decimal",
+            FIELD_TYPE.LONGLONG: "bigint",
+            FIELD_TYPE.INT24: "int",
+            FIELD_TYPE.DATE: "date",
+            FIELD_TYPE.NEWDATE: "date",
+            FIELD_TYPE.DATETIME: "datetime",
+            FIELD_TYPE.STRING: "varchar",
+            FIELD_TYPE.VARCHAR: "varchar",
+            FIELD_TYPE.JSON: "json",
+        }
+        return type_code_mapping.get(type_code, "string")

recurvedata/connectors/connectors/tencent_cos.py

@@ -0,0 +1,40 @@
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.object_store import ObjectStoreMixin
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "cos"
+UI_CONNECTION_TYPE = "Tencent COS"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class COS(ObjectStoreMixin):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+    setup_extras_require = []
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "secret_id": {"type": "string", "title": _l("Tencent Cloud API Secret ID")},
+            "secret_key": {"type": "string", "title": _l("Tencent Cloud API Secret Key")},
+            "region": {"type": "string", "title": _l("COS Region"), "default": "ap-guangzhou"},
+            "bucket": {"type": "string", "title": _l("Bucket Name")},
+        },
+        "order": ["secret_id", "secret_key", "region", "bucket"],
+        "required": ["secret_id", "secret_key", "region", "bucket"],
+        "secret": ["secret_key"],
+    }
+
+    def init_connection(self, conf):
+        self.connector = None  # todo
+
+    def test_connection(self):
+        # todo
+        pass
+
+    juice_sync_able = True
+
+    def juice_sync_path(self, path: str) -> str:
+        return f"cos://{path}"  # todo

recurvedata/connectors/connectors/tidb.py

@@ -0,0 +1,49 @@
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.dbapi import DBAPIBase, with_ssh_tunnel
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "tidb"
+UI_CONNECTION_TYPE = "TiDB"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class TidbConnector(DBAPIBase):
+    SYSTEM_DATABASES = ["information_schema", "mysql", "performance_schema", "sys"]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+    setup_extras_require = ["mysql-connector-python"]
+    # pip install git+https://github.com/pingcap/sqlalchemy-tidb.git@main
+    # manually run setup.py and upload to Recurvedata pypi
+
+    driver = "mysql+mysqlconnector"
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {"type": "string", "title": _l("Host Address"), "default": "127.0.0.1"},
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 4000,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+                "description": _l("The name of the database to connect to"),
+            },
+        },
+        "order": ["host", "port", "user", "password", "database"],
+        "required": ["host"],
+        "secret": ["password"],
+    }
+
+    @with_ssh_tunnel
+    def get_columns(self, table, database=None):
+        database = database or self.database
+        column_dcts = self.inspector.get_columns(table, schema=database)
+        for dct in column_dcts:
+            dct["type"] = self._extract_column_name(dct["type"]).lower()
+        return column_dcts