recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/connectors/connectors/mongo.py

@@ -0,0 +1,79 @@
+try:
+    from pymongo import MongoClient
+except ImportError:
+    pass
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.dbapi import DBAPIBase, with_ssh_tunnel
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "mongodb"
+UI_CONNECTION_TYPE = "MongoDB"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class MongoDB(DBAPIBase):
+    SYSTEM_DATABASES = [
+        "admin",
+        "config",
+        "local",
+    ]
+    setup_extras_require = ["pymongo"]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 27017,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "authSource": {
+                "type": "string",
+                "title": _l("Authentication Database"),
+                "default": "admin",
+            },
+        },
+        "order": ["host", "port", "user", "password", "authSource"],
+        "required": ["host", "port", "user", "password"],
+        "secret": ["password"],
+    }
+
+    def test_connection(self):
+        client = MongoClient(
+            host=self.conf["host"],
+            port=self.conf["port"],
+            username=self.conf["user"],
+            password=self.conf["password"],
+            authSource=self.conf["authSource"],
+            serverSelectionTimeoutMS=5000,
+        )
+        client.admin.command("ping")
+
+    @classmethod
+    def get_sql_operator_types(cls) -> list[str]:
+        return []
+
+    @with_ssh_tunnel
+    def get_databases(self) -> list[str]:
+        client = MongoClient(
+            host=self.conf["host"],
+            port=self.conf["port"],
+            username=self.conf["user"],
+            password=self.conf["password"],
+            authSource=self.conf["authSource"],
+            serverSelectionTimeoutMS=5000,
+        )
+        databases = client.list_database_names()
+        return [d for d in databases if d not in self.SYSTEM_DATABASES]

recurvedata/connectors/connectors/mssql.py

@@ -0,0 +1,131 @@
+from sqlalchemy.engine import URL
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.dbapi import DBAPIBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "mssql"
+UI_CONNECTION_TYPE = "Microsoft SQL Server"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class MssqlConnector(DBAPIBase):
+    SYSTEM_DATABASES = ["master", "model", "msdb", "tempdb"]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+
+    setup_extras_require = ["pyodbc"]
+    driver = "mssql+pyodbc"
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 1433,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+                "description": _l("The name of the database to connect to"),
+                "default": "default",
+            },
+            "odbc_driver": {
+                "type": "string",
+                "title": _l("ODBC Driver"),
+                "default": "ODBC Driver 18 for SQL Server",
+            },
+            "encrypt": {
+                "type": "boolean",
+                "title": _l("Encrypt Connection"),
+                "description": _l("Whether to encrypt the connection"),
+                "default": True,
+            },
+            "trust_server_certificate": {
+                "type": "boolean",
+                "title": _l("Trust Server Certificate"),
+                "default": True,
+            },
+        },
+        "order": ["host", "port", "user", "password", "database", "odbc_driver", "encrypt", "trust_server_certificate"],
+        "required": [
+            "host",
+        ],
+        "secret": ["password"],
+    }
+
+    @property
+    def odbc_driver(self):
+        return self.conf["odbc_driver"]
+
+    @property
+    def sqlalchemy_url(self):
+        return URL(
+            self.driver,
+            self.user,
+            self.password,
+            self.host,
+            self.port,
+            self.database,
+            query={
+                "driver": self.odbc_driver,
+                "TrustServerCertificate": "yes" if self.conf.get("trust_server_certificate", False) else "no",
+                "Encrypt": "yes" if self.conf.get("encrypt", True) else "no",
+            },
+        )
+
+    @classmethod
+    def get_sql_operator_types(cls):
+        return [cls.connection_type, "azure_mssql"]
+
+    @classmethod
+    def order_sql(cls, sql: str, orders: list[dict[str, str]] | None = None, return_sql: bool = True) -> str:
+        """Format SQL with ORDER BY clause for MSSQL.
+
+        Args:
+            sql: The SQL query to add ORDER BY to
+            orders: List of dicts with column and order direction
+            return_sql: Whether to return the formatted SQL
+
+        Returns:
+            SQL query with ORDER BY clause
+        """
+        if not orders:
+            return sql
+
+        order_clauses = [f"{order['field']} {order['order']}" for order in orders]
+        order_by = f" ORDER BY {', '.join(order_clauses)}"
+        return f"{sql}{order_by}"
+
+    @classmethod
+    def limit_sql(
+        cls, sql: str, limit: int = 100, orders: list[dict[str, str]] | None = None, offset: int | None = None
+    ) -> str:
+        """Add TOP/OFFSET-FETCH clause to SQL query for MSSQL.
+
+        Args:
+            sql: The SQL query to add limit to
+            limit: Maximum number of rows to return
+            orders: List of order by clauses
+            offset: Number of rows to skip
+
+        Returns:
+            SQL query with TOP/OFFSET-FETCH clause
+        """
+        sub_query_name = "_recurve_limit_subquery"
+        sql = cls.order_sql(sql, orders)
+
+        if offset:
+            return f"{sql} OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY"
+
+        return f"SELECT TOP {limit} * FROM ({sql}) AS {sub_query_name}"

recurvedata/connectors/connectors/mysql.py

@@ -0,0 +1,191 @@
+from functools import cached_property
+from typing import Any
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.datasource import DataSourceWrapper
+from recurvedata.connectors.dbapi import DBAPIBase, with_ssh_tunnel
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "mysql"
+UI_CONNECTION_TYPE = "MySQL"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class MysqlConnector(DBAPIBase):
+    SYSTEM_DATABASES = ["information_schema", "mysql", "performance_schema", "sys"]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+
+    setup_extras_require = ["pymysql"]
+    driver = "mysql+pymysql"
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 3306,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+                "description": _l("The name of the database to connect to"),
+            },
+            # 'ssh_tunnel': SSH_TUNNEL_CONFIG_SCHEMA
+        },
+        "order": [
+            "host",
+            "port",
+            "user",
+            "password",
+            "database",
+        ],
+        "required": [
+            "host",
+        ],
+        "secret": ["password"],
+    }
+
+    column_type_mapping = {
+        "integer": ["mediumint", "year", "long"],
+        "string": ["tinytext", "mediumtext", "longtext", "enum", "set", "blob"],
+        "binary": ["tinyblob", "mediumblob", "longblob"],
+    }
+
+    available_column_types = DBAPIBase.available_column_types + [
+        # Integer types
+        "tinyint",
+        "tinyint unsigned",
+        "bit",
+        "mediumint",
+        "int unsigned",
+        "integer",  # alias for int
+        "bigint unsigned",
+        # Floating-point types
+        "numeric",  # alias for decimal
+        "fixed",  # alias for decimal
+        "double precision",  # alias for double
+        "float4",  # alias for float
+        "float8",  # alias for double
+        # String types
+        "tinytext",
+        "text",
+        "mediumtext",
+        "longtext",
+        "long",  # alias for mediumtext
+        "character",  # alias for char
+        "long varchar",  # alias for mediumtext
+        "enum",
+        "set",
+        # Binary string types
+        "binary",
+        "varbinary",
+        "tinyblob",
+        "blob",
+        "mediumblob",
+        "longblob",
+        # Date and Time types
+        "datetime",
+        "time",
+        "year",
+        # Boolean type
+        "bool",  # alias for boolean
+    ]
+
+    @with_ssh_tunnel
+    def get_columns(self, table, database=None):
+        database = database or self.database
+        query = f"""
+        SELECT column_name,
+            data_type,
+            is_nullable,
+            column_default,
+            column_comment
+            FROM INFORMATION_SCHEMA.COLUMNS
+            WHERE TABLE_SCHEMA = '{database}' AND TABLE_NAME = '{table}';
+        """
+        result = self.fetchall(query)
+        column_metas = []
+        for row in result:
+            column_metas.append(
+                {
+                    "name": row[0],
+                    "type": row[1].lower() if row[1] else "",
+                    "nullable": row[2] == "YES",
+                    "default": row[3],
+                    "comment": row[4],
+                }
+            )
+        return column_metas
+
+    @cached_property
+    @with_ssh_tunnel
+    def type_code_mapping(self) -> dict:
+        """
+        type_code from sqlalchemy's cursor.description -> database's dialect data type name
+        """
+        return {}
+
+    def sqlalchemy_column_type_code_to_name(self, type_code: Any, size: int | None = None) -> str:
+        from pymysql.constants import FIELD_TYPE
+
+        mapping = {type_code: name.lower() for name, type_code in vars(FIELD_TYPE).items() if not name.startswith("__")}
+        type_mapping = {
+            FIELD_TYPE.TINY: "tinyint",
+            FIELD_TYPE.SHORT: "smallint",
+            FIELD_TYPE.LONG: "int",
+            FIELD_TYPE.LONGLONG: "bigint",
+            FIELD_TYPE.INT24: "mediumint",
+            FIELD_TYPE.NEWDECIMAL: "decimal",
+            FIELD_TYPE.TINY_BLOB: "tinyblob",
+            FIELD_TYPE.MEDIUM_BLOB: "mediumblob",
+            FIELD_TYPE.LONG_BLOB: "longblob",
+            FIELD_TYPE.VAR_STRING: "varchar",
+            FIELD_TYPE.STRING: "char",
+        }
+        return type_mapping.get(type_code, mapping.get(type_code, "varchar"))
+
+    @classmethod
+    def order_sql(cls, sql: str, orders: list[dict[str, str]] = None):
+        base_sql = f"SELECT * FROM ({sql}) AS _recurve_limit_subquery"
+        if orders:
+            order_clauses = [f"{order['field']} {order['order']}" for order in orders]
+            base_sql += " ORDER BY " + ", ".join(order_clauses)
+        return base_sql
+
+    @classmethod
+    def limit_sql(cls, sql: str, limit: int = 100, orders: list[dict[str, str]] | None = None, offset: int = 0) -> str:
+        """
+        the sqlglot will convert `timestamp` to `datetime`,
+        which cause this sql: `cast(field as timestamp) as field` to be error in dbt build but success in preview.
+        args:
+            sql: the sql to be limited
+            limit: the limit of the sql
+            orders: the orders of the sql(list[dict[str, str]]), each dict contains `field` and `order`, used in data service preview
+        """
+        base_sql = cls.order_sql(sql, orders)
+        if offset:
+            return f"{base_sql} LIMIT {offset}, {limit}"
+
+        return f"{base_sql} LIMIT {limit}"
+
+    def convert_config_to_cube_config(
+        self, database: str, schema: str = None, datasource: DataSourceWrapper = None
+    ) -> dict:
+        return {
+            "type": "mysql",
+            "host": self.host,
+            "port": self.port,
+            "user": datasource.user,
+            "password": datasource.password,
+            "database": database or self.database,
+        }

recurvedata/connectors/connectors/n8n.py

@@ -0,0 +1,141 @@
+import base64
+from contextlib import contextmanager
+from urllib.parse import urlparse
+
+import httpx
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.connectors.proxy import HTTP_PROXY_CONFIG_SCHEMA, HttpProxyMixin
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "n8n"
+UI_CONNECTION_TYPE = "n8n"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class N8N(HttpProxyMixin, RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "title": _l("API Address"),
+                "description": _l("The URL of the n8n API, e.g. https://localhost:5678/api/v1"),
+            },
+            "api_key": {"type": "string", "title": _l("API KEY")},
+            "timeout": {
+                "type": "number",
+                "title": _l("Timeout"),
+                "description": _l("The timeout of the n8n API, e.g. 60"),
+                "default": 60,
+            },
+            "webhook_credential": {
+                "type": "object",
+                "title": _l("Webhook Trigger Node Credential"),
+                "description": _l("The credential of the n8n webhook trigger node"),
+                "properties": {
+                    "credential_type": {
+                        "type": "string",
+                        "title": _l("Credential Type"),
+                        "enum": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
+                        "enumNames": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
+                        "default": "None",
+                    },
+                    "basic_auth": {
+                        "ui:hidden": '{{parentFormData.credential_type !== "Basic Auth"}}',
+                        "type": "object",
+                        "title": _l("Basic Auth"),
+                        "description": _l("The basic auth of the n8n webhook trigger node"),
+                        "properties": {
+                            "username": {"type": "string", "title": _l("Username")},
+                            "password": {"type": "string", "title": _l("Password")},
+                        },
+                    },
+                    "header_auth": {
+                        "ui:hidden": '{{parentFormData.credential_type !== "Header Auth"}}',
+                        "type": "object",
+                        "title": _l("Header Auth"),
+                        "description": _l("The header auth of the n8n webhook trigger node"),
+                        "properties": {
+                            "header_name": {"type": "string", "title": _l("Header Name")},
+                            "header_value": {"type": "string", "title": _l("Header Value")},
+                        },
+                    },
+                    "jwt_auth": {
+                        "ui:hidden": '{{parentFormData.credential_type !== "JWT Auth"}}',
+                        "type": "object",
+                        "title": _l("JWT Auth"),
+                        "description": _l("The jwt auth of the n8n webhook trigger node"),
+                        "properties": {
+                            "jwt_token": {"type": "string", "title": _l("JWT Token")},
+                        },
+                    },
+                },
+                "order": ["credential_type", "basic_auth", "header_auth", "jwt_auth"],
+            },
+            "proxies": HTTP_PROXY_CONFIG_SCHEMA["proxies"],
+        },
+        "order": ["url", "api_key", "timeout", "webhook_credential", "proxies"],
+        "required": ["url", "api_key"],
+        "secret": [
+            "api_key",
+            "webhook_credential.basic_auth.password",
+            "webhook_credential.header_auth.header_value",
+            "webhook_credential.jwt_auth.jwt_token",
+        ],
+    }
+
+    def test_connection(self):
+        pass
+
+    @contextmanager
+    def _n8n_client(self) -> httpx.Client:
+        with self._init_proxy_manager():
+            yield httpx.Client(
+                base_url=f"{self.url}", headers={"X-N8N-API-KEY": f"{self.api_key}"}, timeout=self.timeout
+            )
+
+    def get_workflows(self) -> list[dict]:
+        path = "/workflows"
+        workflows = []
+        cursor = None
+        with self._n8n_client() as client:
+            response = client.get(path)
+            workflows.extend(response.json()["data"])
+            if response.json()["nextCursor"] and response.json()["nextCursor"] != cursor:
+                cursor = response.json()["nextCursor"]
+                while cursor:
+                    response = client.get(path, params={"cursor": cursor})
+                    workflows.extend(response.json()["data"])
+                    cursor = response.json()["nextCursor"]
+            return workflows
+
+    def _trigger_workflow_via_webhook(self, webhook_id: str, payload: dict) -> dict:
+        main_url = f"{urlparse(self.url).scheme}://{urlparse(self.url).netloc}"
+        webhook_url = f"{main_url}/webhook/{webhook_id}"
+        headers = {}
+        credential_type = self.webhook_credential.get("credential_type")
+        basic_auth = self.webhook_credential.get("basic_auth", {})
+        header_auth = self.webhook_credential.get("header_auth", {})
+        jwt_auth = self.webhook_credential.get("jwt_auth", {})
+        with self._init_proxy_manager():
+            if credential_type == "Basic Auth":
+                username = basic_auth.get("username", "")
+                password = basic_auth.get("password", "")
+                headers["Authorization"] = f'Basic {base64.b64encode(f"{username}:{password}".encode()).decode()}'
+            elif credential_type == "Header Auth":
+                header_name = header_auth.get("header_name", "")
+                header_value = header_auth.get("header_value", "")
+                headers[header_name] = header_value
+            elif credential_type == "JWT Auth":
+                jwt_token = jwt_auth.get("jwt_token", "")
+                headers["Authorization"] = f"Bearer {jwt_token}"
+            response = httpx.post(url=webhook_url, headers=headers, timeout=self.timeout, json=payload)
+
+            return response.json()

recurvedata/connectors/connectors/oss.py

@@ -0,0 +1,74 @@
+try:
+    from ossfs import OSSFileSystem
+except ImportError:
+    OSSFileSystem = None
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.object_store import ObjectStoreMixin
+from recurvedata.connectors.proxy import HTTP_PROXY_CONFIG_SCHEMA, HttpProxyMixin
+from recurvedata.connectors.utils import juice_sync_process_special_character_within_secret
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "oss"
+UI_CONNECTION_TYPE = "Aliyun OSS"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class OSS(HttpProxyMixin, ObjectStoreMixin):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+    setup_extras_require = ["ossfs"]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "access_key_id": {
+                "type": "string",
+                "title": _l("Aliyun Access Key ID"),
+                "description": _l("The AccessKey ID for authenticating with Aliyun OSS"),
+            },
+            "secret_access_key": {
+                "type": "string",
+                "title": _l("Aliyun Secret Access Key"),
+                "description": _l("The AccessKey Secret for authenticating with Aliyun OSS"),
+            },
+            "endpoint": {
+                "type": "string",
+                "title": _l("OSS Endpoint"),
+                "description": _l("The endpoint URL for your OSS bucket region"),
+                "default": "oss-cn-hangzhou.aliyuncs.com",
+            },
+            "bucket": {"type": "string", "title": _l("Bucket Name")},
+            "proxies": HTTP_PROXY_CONFIG_SCHEMA["proxies"],
+        },
+        "order": [
+            "access_key_id",
+            "secret_access_key",
+            "endpoint",
+            "bucket",
+            "proxies",
+        ],
+        "required": ["access_key_id", "secret_access_key", "endpoint", "bucket"],
+        "secret": ["secret_access_key"],
+    }
+
+    def init_connection(self, conf) -> OSSFileSystem:
+        con = OSSFileSystem(
+            key=conf["access_key_id"],
+            secret=conf["secret_access_key"],
+            endpoint=conf["endpoint"],
+        )
+        self.connector = con
+        return con
+
+    juice_sync_able = True
+
+    def juice_sync_path(self, path: str) -> tuple[str, str]:
+        secret_part = f"{self.access_key_id}:{self.secret_access_key}"
+        secret_part = juice_sync_process_special_character_within_secret(secret_part)
+        common_part = f'{self.bucket}.{self.endpoint}/{path.lstrip("/")}'
+        secret_path = f"oss://{secret_part}@{common_part}"
+        non_secret_path = f"oss://{common_part}"
+        return secret_path, non_secret_path

recurvedata/connectors/connectors/owncloud.py

@@ -0,0 +1,36 @@
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.consts import ConnectionCategory, ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "owncloud"
+UI_CONNECTION_TYPE = "OwnCloud"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class OwnCloud(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    category = [ConnectionCategory.STORAGE]
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "url": {"type": "string", "title": _l("Host URL")},
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "dav_endpoint_version": {
+                "type": "number",
+                "title": _l("WebDAV Endpoint Version"),
+                "default": 1,
+            },
+        },
+        "order": ["url", "user", "password", "dav_endpoint_version"],
+        "required": ["url", "user", "password"],
+        "secret": ["password"],
+    }
+
+    def test_connection(self):
+        # todo: 暂不校验
+        pass