recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/connectors/connectors/impala.py

@@ -0,0 +1,210 @@
+from functools import cached_property
+
+try:
+    import re
+
+    from pyhive.sqlalchemy_hive import HiveDialect, exc
+    from sqlalchemy import text
+
+    def _get_table_columns(self, connection, table_name, schema):
+        full_table = table_name
+        if schema:
+            full_table = schema + "." + table_name
+        # TODO using TGetColumnsReq hangs after sending TFetchResultsReq.
+        # Using DESCRIBE works but is uglier.
+        try:
+            # This needs the table name to be unescaped (no backticks).
+            rows = connection.execute(text("DESCRIBE {}".format(full_table))).fetchall()
+        except exc.OperationalError as e:
+            # Does the table exist?
+            regex_fmt = r"TExecuteStatementResp.*SemanticException.*Table not found {}"
+            regex = regex_fmt.format(re.escape(full_table))
+            if re.search(regex, e.args[0]):
+                raise exc.NoSuchTableError(full_table)
+            else:
+                raise
+        else:
+            # Hive is stupid: this is what I get from DESCRIBE some_schema.does_not_exist
+            regex = r"Table .* does not exist"
+            if len(rows) == 1:
+                # recurvedata changed
+                if "name" not in rows[0].keys():  # hive
+                    if re.match(regex, rows[0].col_name):
+                        raise exc.NoSuchTableError(full_table)
+                else:
+                    if re.match(regex, rows[0].name):  # impala
+                        raise exc.NoSuchTableError(full_table)
+                # recurvedata changed finish pyhive==0.6.5
+            return rows
+
+    HiveDialect._get_table_columns = _get_table_columns
+except ImportError:
+    pass
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.const import ENV_VAR_DBT_PASSWORD, ENV_VAR_DBT_USER
+from recurvedata.connectors.dbapi import DBAPIBase, with_ssh_tunnel
+from recurvedata.consts import ConnectionCategory, ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "impala"
+UI_CONNECTION_TYPE = "Apache Impala"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class ImpalaConnector(DBAPIBase):
+    SYSTEM_DATABASES = [
+        "information_schema",
+        "_impala_builtins",
+    ]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    setup_extras_require = ["PyHive", "thrift-sasl"]
+    driver = "hive"  # todo: 先用 hive 的
+    category = [ConnectionCategory.WAREHOUSE]
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 21050,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database"),
+                "description": _l("The name of the database to connect to"),
+                "default": "default",
+            },
+            "hdfs_options": {
+                "type": "object",
+                "title": _l("HDFS Configuration"),
+                "description": _l("Configuration options for HDFS connection"),
+                "properties": {
+                    "host": {
+                        "type": "string",
+                        "title": _l("Host"),
+                        "description": _l("HDFS namenode hostname or IP address"),
+                    },
+                    "port": {
+                        "type": "number",
+                        "title": _l("Port Number"),
+                        "description": _l("HDFS namenode port number"),
+                        "default": 50070,
+                    },
+                    "user": {"type": "string", "title": _l("Username")},
+                },
+                "order": ["host", "port", "user"],
+            },
+            "auth_mechanism": {
+                "type": "string",
+                "title": _l("Authentication Mechanism"),
+                "description": _l("Impala authentication mechanism (e.g. PLAIN, GSSAPI, LDAP)"),
+                "default": "PLAIN",
+            },
+            "auth": {
+                "type": "string",
+                "title": _l("Authentication Type"),
+                "default": "LDAP",
+            },
+            "use_http_transport": {
+                "type": "boolean",
+                "title": _l("Use HTTP Transport"),
+                "default": True,
+            },
+            "use_ssl": {
+                "type": "boolean",
+                "title": _l("Use SSL"),
+                "default": True,
+            },
+            "http_path": {
+                "type": "string",
+                "title": _l("HTTP Path"),
+                "default": "",
+            },
+        },
+        "order": [
+            "host",
+            "port",
+            "user",
+            "password",
+            "database",
+            "hdfs_options",
+            "auth",
+            "auth_mechanism",
+            "use_http_transport",
+            "use_ssl",
+            "http_path",
+        ],
+        "required": ["host", "port"],
+        "secret": ["password"],
+    }
+
+    @property
+    def connect_args(self):
+        if not self.password and not self.user:
+            return {"auth": "NOSASL"}
+        if self.password:
+            return {"auth": "LDAP"}  # 先粗暴处理
+        if self.auth == "LDAP":
+            return {"auth": "LDAP"}  # todo
+        return {}
+
+    # generate_ddl todo: stored as parquet
+
+    def _extract_column_name(self, column_type):
+        visit_type = column_type.__visit_name__
+        if visit_type == "type_decorator":
+            return column_type.impl.__visit_name__
+        return visit_type
+
+    @with_ssh_tunnel
+    def get_tables(self, database: str = None):
+        database = database or self.database
+        result = self.fetchall(f"SHOW TABLES IN {database}")
+        return [r[0] for r in result]
+
+    @with_ssh_tunnel
+    def get_views(self, database: str = None):
+        database = database or self.database
+        result = self.fetchall(f"SHOW VIEWS IN {database}")
+        return [r[0] for r in result]
+
+    def get_columns(self, table, database=None):
+        database = database or self.database
+        column_dcts = self.inspector.get_columns(self.format_key(table), schema=self.format_key(database))
+        for dct in column_dcts:
+            dct["type"] = self._extract_column_name(dct["type"]).lower()
+        return column_dcts
+
+    def convert_config_to_dbt_profile(self, database: str, schema: str = None) -> dict:
+        return {
+            "type": "impala",
+            "host": self.host,
+            "http_path": self.http_path,
+            "port": self.port,
+            "auth_type": self.auth.lower(),
+            "use_http_transport": self.use_http_transport,
+            "use_ssl": self.use_ssl,
+            "username": ENV_VAR_DBT_USER,
+            "password": ENV_VAR_DBT_PASSWORD,
+            "schema": database or self.database,
+            "threads": 10,
+        }
+
+    @cached_property
+    @with_ssh_tunnel
+    def type_code_mapping(self) -> dict:
+        return {}
+
+    def sqlalchemy_column_type_code_to_name(self, code: str, size: int | None = None) -> str:
+        return code.lower()

recurvedata/connectors/connectors/jenkins.py

@@ -0,0 +1,51 @@
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.utils.imports import MockModule
+
+try:
+    import jenkins
+except ImportError:
+    jenkins = MockModule("jenkins")
+
+CONNECTION_TYPE = "jenkins"
+UI_CONNECTION_TYPE = "Jenkins"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class Jenkins(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    setup_extras_require = [
+        "python-jenkins",
+    ]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "url": {"type": "string", "title": "url"},
+            "user": {"type": "string", "title": "Username"},
+            "password": {"type": "string", "title": "Password"},
+        },
+        "order": [
+            "url",
+            "user",
+            "password",
+        ],
+        "required": [
+            "url",
+            "user",
+            "password",
+        ],
+        "secret": ["password"],
+    }
+
+    def __init__(self, conf: dict, *args, **kwargs):
+        super().__init__(conf, *args, **kwargs)
+        self.conf = conf
+        self.connector = self.init_connection(conf)
+
+    def init_connection(self, conf):
+        return jenkins.Jenkins(url=conf["url"], username=conf["user"], password=conf["password"])
+
+    def test_connection(self):
+        self.connector.get_whoami()

recurvedata/connectors/connectors/mail.py

@@ -0,0 +1,89 @@
+from smtplib import SMTP, SMTP_SSL
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "mail"
+UI_CONNECTION_TYPE = "Mail"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class Mail(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {"type": "string", "title": _l("SMTP Server Address")},
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 465,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "reply_to": {
+                "type": "string",
+                "title": _l("Reply-To Address"),
+                "description": _l("Email address that recipients will reply to"),
+            },
+            "mail_from": {
+                "type": "string",
+                "title": _l("From Address"),
+                "description": _l("Email address that appears in the From field"),
+            },
+            "ssl": {
+                "type": "boolean",
+                "title": _l("Use SSL/TLS"),
+                "description": _l("Enable SSL/TLS encryption for secure email transmission"),
+                "default": True,
+            },
+            "timeout": {
+                "type": "number",
+                "title": _l("Connection Timeout"),
+                "description": _l("Maximum time in seconds to wait for server connection"),
+                "default": 180,
+            },
+        },
+        "order": [
+            "host",
+            "port",
+            "user",
+            "password",
+            "reply_to",
+            "mail_from",
+            "ssl",
+            "timeout",
+        ],
+        "required": ["host", "port", "user", "password"],
+        "secret": ["password"],
+    }
+
+    def __init__(self, conf: dict, *args, **kwargs):
+        self.conf = conf
+        self.connector, self.connector_err = self.init_connection(conf)
+
+    def init_connection(self, conf: dict):
+        try:
+            smtp_class = SMTP if not conf.get("ssl") else SMTP_SSL
+            smtp = smtp_class(host=conf["host"], port=conf["port"], timeout=conf.get("timeout", 180))
+            if conf.get("user") and conf.get("password"):
+                smtp.login(conf["user"], conf["password"])
+            return smtp, None
+        except Exception as e:
+            return None, e
+
+    def test_connection(self):
+        if self.connector_err:
+            raise ValueError(f"Failed to connect: {self.connector_err}")
+        if not self.connector:
+            raise ValueError("SMTP connection not initialized")
+
+        try:
+            res = self.connector.noop()
+            if res[0] != 250:
+                raise ValueError(f"SMTP server returned unexpected response: {res}")
+        except Exception as e:
+            raise ValueError(f"SMTP connection test failed: {str(e)}")

recurvedata/connectors/connectors/microsoft_fabric.py

@@ -0,0 +1,284 @@
+import datetime
+import logging
+import uuid
+from enum import StrEnum
+from typing import Any
+
+from sqlalchemy.engine import URL
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.connectors.mssql import MssqlConnector
+from recurvedata.consts import ConnectionCategory
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "microsoft_fabric"
+UI_CONNECTION_TYPE = "Microsoft Fabric"
+
+
+class AuthMethod(StrEnum):
+    """Microsoft Fabric authentication methods"""
+
+    SERVICE_PRINCIPAL = "ActiveDirectoryServicePrincipal"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class MsFabricConnector(MssqlConnector):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    category = [ConnectionCategory.WAREHOUSE]
+    SYSTEM_DATABASES = [
+        "information_schema",
+        "sys",
+        "db_owner",
+        "db_datareader",
+        "db_datawriter",
+        "db_ddladmin",
+        "db_securityadmin",
+        "db_accessadmin",
+        "db_backupoperator",
+        "db_denydatawriter",
+        "db_denydatareader",
+        "_rsc",
+        "queryinsights",
+        "guest",
+    ]
+
+    available_column_types = [
+        "bit",
+        "smallint",
+        "int",
+        "bigint",
+        "decimal",
+        "numeric",
+        "float",
+        "real",
+        "date",
+        "time",
+        "datetime2",
+        "char",
+        "varchar",
+        "varbinary",
+        "uniqueidentifier",
+    ]
+
+    column_type_mapping = {
+        "integer": ["smallint", "int", "bigint"],
+        "float": ["real", "float"],
+        "datetime": ["datetime2"],
+        "string": ["char", "varchar"],
+        "binary": ["varbinary", "uniqueidentifier"],
+    }
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 1433,
+            },
+            "authentication": {
+                "type": "string",
+                "title": _l("Authentication Method"),
+                "default": "ActiveDirectoryServicePrincipal",
+                "ui:options": {
+                    "disabled": True,
+                },
+            },
+            "tenant_id": {
+                "type": "string",
+                "title": _l("Tenant ID"),
+            },
+            "client_id": {
+                "type": "string",
+                "title": _l("Client ID"),
+            },
+            "client_secret": {
+                "type": "string",
+                "title": _l("Client Secret"),
+            },
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+            },
+            "odbc_driver": {
+                "type": "string",
+                "title": _l("ODBC Driver"),
+                "default": "ODBC Driver 18 for SQL Server",
+                "ui:options": {
+                    "disabled": True,
+                },
+            },
+            "encrypt": {
+                "type": "boolean",
+                "title": _l("Encrypt Connection"),
+                "description": _l("Whether to encrypt the connection"),
+                "default": True,
+            },
+            "trust_server_certificate": {
+                "type": "boolean",
+                "title": _l("Trust Server Certificate"),
+                "default": True,
+            },
+            "blob_options": {
+                "type": "object",
+                "title": _l("Azure Blob Storage Options"),
+                "properties": {
+                    "account_name": {"type": "string", "title": _l("Storage Account Name")},
+                    "endpoint_suffix": {"type": "string", "title": _l("Endpoint Suffix")},
+                    "container_name": {"type": "string", "title": _l("Container Name")},
+                    "sas_token": {"type": "string", "title": _l("SAS Token")},
+                },
+                "order": ["account_name", "endpoint_suffix", "container_name", "sas_token"],
+            },
+        },
+        "order": [
+            "host",
+            "port",
+            "authentication",
+            "tenant_id",
+            "client_id",
+            "client_secret",
+            "database",
+            "odbc_driver",
+            "encrypt",
+            "trust_server_certificate",
+            "blob_options",
+        ],
+        "required": ["host", "port", "tenant_id", "client_id", "client_secret", "database"],
+        "secret": ["client_secret", "blob_options.sas_token"],
+    }
+
+    INT_TYPE_MAPPING = {
+        range(0, 4): "tinyint",
+        range(4, 6): "smallint",
+        range(6, 11): "int",
+        range(11, 20): "bigint",
+    }
+
+    BASE_TYPE_MAPPING = {
+        bool: "bit",
+        float: "float",
+        datetime.datetime: "datetime2",
+        str: "varchar",
+        bytes: "varbinary",
+        uuid.UUID: "uniqueidentifier",
+    }
+
+    @property
+    def sqlalchemy_url(self):
+        query = {
+            "driver": self.odbc_driver,
+            "authentication": AuthMethod.SERVICE_PRINCIPAL.value,
+            "Encrypt": "yes" if self.conf.get("encrypt", True) else "no",
+            "TrustServerCertificate": "yes" if self.conf.get("trust_server_certificate", True) else "no",
+            "Tenant Id": self.conf["tenant_id"],
+        }
+
+        url = URL(
+            self.driver,
+            self.conf["client_id"],
+            self.conf["client_secret"],
+            self.conf["host"],
+            self.conf["port"],
+            self.conf["database"],
+            query=query,
+        )
+
+        # Debug logging
+        logging.info(f"Connection URL (without credentials): {url.render_as_string(hide_password=True)}")
+        return url
+
+    def convert_config_to_dbt_profile(self, database: str, schema: str = None) -> dict[str, Any]:
+        return {
+            "server": self.conf["host"],
+            "port": self.conf["port"],
+            "client_id": self.conf["client_id"],
+            "client_secret": self.conf["client_secret"],
+            "database": database or self.database,
+            "type": "fabric",
+            "authentication": "ServicePrincipal",
+            "tenant_id": self.conf["tenant_id"],
+            "schema": schema or database or self.database,
+            "driver": self.odbc_driver,
+        }
+
+    def sqlalchemy_column_type_code_to_name(self, type_code: Any, size: int | None = None) -> str:
+        """
+        Map SQL Server/Microsoft Fabric type codes to their corresponding type names.
+        Reference: https://learn.microsoft.com/en-us/fabric/data-warehouse/data-types
+
+        Microsoft Fabric supports a subset of T-SQL data types, including:
+        - Exact numerics: bit, smallint, int, bigint, decimal/numeric
+        - Approximate numerics: float, real
+        - Date and time: date, time, datetime2
+        - Character strings: char, varchar
+        - Binary strings: varbinary, uniqueidentifier
+        """
+        # First try to get the Python type from type_code
+        py_type = type(type_code) if type_code is not None else None
+
+        # If type_code itself is a type (like int, str, etc), use it directly
+        if isinstance(type_code, type):
+            py_type = type_code
+
+        type_mapping = {
+            bool: "bit",
+            int: "int",
+            float: "float",
+            datetime.datetime: "datetime2",
+            str: "varchar",
+            bytes: "varbinary",
+            uuid.UUID: "uniqueidentifier",
+        }
+
+        if py_type in type_mapping:
+            base_type = type_mapping[py_type]
+            if py_type is int and size is not None:
+                for size_range, type_name in self.INT_TYPE_MAPPING.items():
+                    if size in size_range:
+                        return type_name
+            return base_type
+
+        return "varchar"
+
+    def set_env_when_get_dbt_connection(self):
+        pass
+
+    @classmethod
+    def limit_sql(
+        cls, sql: str, limit: int = 100, orders: list[dict[str, str]] | None = None, offset: int | None = None
+    ) -> str:
+        """Add pagination to SQL query for Microsoft Fabric.
+
+        Args:
+            sql: The SQL query to add limit to
+            limit: Maximum number of rows to return
+            orders: List of order by clauses
+            offset: Number of rows to skip
+
+        Returns:
+            SQL query with pagination
+        """
+        # dbt model with `ephemeral` will automatically add `__dbt__cte__` to the query
+        # which is not supported by Microsoft Fabric
+        # so we need to handle it separately
+        if "__dbt__cte__" not in sql and not sql.upper().strip().startswith("WITH"):
+            return super().limit_sql(sql, limit, orders, offset)
+
+        sql = cls.order_sql(sql, orders)
+
+        # Build final query with CTE
+        sub_query_name = "_recurve_limit_subquery"
+        base_sql = f"WITH {sub_query_name} AS ({sql})"
+
+        if offset:
+            base_sql = f"{base_sql} ORDER BY (SELECT NULL)"
+            return f"{base_sql} SELECT * FROM {sub_query_name} OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY"
+
+        return f"{base_sql} SELECT TOP {limit} * FROM {sub_query_name}"