recurvedata-lib 0.1.487__py2.py3-none-any.whl

recurvedata/connectors/connectors/phoenix.py

@@ -0,0 +1,36 @@
+from sqlalchemy.engine import URL
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.dbapi import DBAPIBase
+from recurvedata.consts import ConnectorGroup
+
+CONNECTION_TYPE = "phoenix"
+UI_CONNECTION_TYPE = "HBase Phoenix"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class PhoenixConnector(DBAPIBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+    setup_extras_require = ["sqlalchemy-phoenix"]
+    driver = "phoenix"
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {"type": "string", "title": "Host Address"},
+            "port": {
+                "type": "number",
+                "title": "Port Number",
+                "default": 8765,
+            },
+        },
+        "order": ["host", "port"],
+        "required": ["host", "port"],
+        "secret": [],
+    }
+
+    @property
+    def sqlalchemy_url(self):
+        return URL(self.driver, host=self.host, port=self.port)

recurvedata/connectors/connectors/postgres.py

@@ -0,0 +1,230 @@
+from functools import cached_property
+from typing import Any
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.const import ENV_VAR_DBT_PASSWORD, ENV_VAR_DBT_USER
+from recurvedata.connectors.datasource import DataSourceWrapper
+from recurvedata.connectors.dbapi import DBAPIBase, with_ssh_tunnel
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "postgres"
+UI_CONNECTION_TYPE = "PostgreSQL"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class PostgresConnector(DBAPIBase):
+    SYSTEM_DATABASES = [
+        "information_schema",
+        "pg_catalog",
+        "pg_global",
+        "pg_statistic",
+        "pg_toast",
+        "pg_temp_1",
+        "pg_temp_2",
+        "pg_toast_temp_1",
+        "pg_toast_temp_2",
+        "pg_type",
+    ]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+    setup_extras_require = ["psycopg2-binary"]
+    driver = "postgresql"
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 5432,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+                "description": _l("The name of the database to connect to"),
+            },
+        },
+        "order": ["host", "port", "user", "password", "database"],
+        "required": [
+            "host",
+            "user",
+            "password",
+        ],
+        "secret": ["password"],
+    }
+
+    column_type_mapping = {
+        "integer": ["int2", "int4", "int8", "serial", "bigserial", "smallserial"],
+        "float": ["float4", "float8", "real", "numeric", "decimal"],
+        "datetime": ["timestamptz"],
+        "time": ["timetz"],
+        "binary": ["bytea"],
+        "string": ["uuid"],
+        "json": ["jsonb"],
+    }
+
+    # Extend base types with PostgreSQL specific types
+    available_column_types = DBAPIBase.available_column_types + [
+        # Numeric types
+        "int2",  # alias for smallint
+        "integer",
+        "int4",  # alias for integer
+        "int8",  # alias for bigint
+        "real",
+        "float4",  # alias for real
+        "double precision",
+        "float8",  # alias for double precision
+        "serial",
+        "serial4",  # alias for serial
+        "bigserial",
+        "serial8",  # alias for bigserial
+        "smallserial",
+        "serial2",  # alias for smallserial
+        "money",
+        "numeric",
+        # Character types
+        "text",
+        "bpchar",  # blank-padded char
+        "character",
+        "character varying",
+        # Date/Time types
+        "timestamp without time zone",
+        "timestamp with time zone",
+        "timestamptz",  # alias for timestamp with time zone
+        "time",
+        "time without time zone",
+        "time with time zone",
+        "timetz",  # alias for time with time zone
+        "interval",
+        # Boolean type
+        "boolean",
+        "bool",  # alias for boolean
+        # Geometric types
+        "point",
+        "line",
+        "lseg",
+        "box",
+        "path",
+        "polygon",
+        "circle",
+        # Network address types
+        "cidr",
+        "inet",
+        "macaddr",
+        "macaddr8",
+        # Binary data
+        "bytea",
+        # UUID type
+        "uuid",
+        # JSON types
+        "jsonb",
+        # XML type
+        "xml",
+        # Bit string
+        "bit",
+        "bit varying",
+        "varbit",
+        # Text search
+        "tsvector",
+        "tsquery",
+        # Range types
+        "int4range",
+        "int8range",
+        "numrange",
+        "tsrange",
+        "tstzrange",
+        "daterange",
+    ]
+
+    def convert_config_to_dbt_profile(self, database: str, schema: str = None) -> dict:
+        return {
+            "host": self.host,
+            "port": self.port,
+            "user": ENV_VAR_DBT_USER,
+            "password": ENV_VAR_DBT_PASSWORD,
+            "dbname": database or self.database,
+            "schema": schema or f"dbt_{database or self.database}",
+            "type": self.connection_type,
+        }
+
+    @with_ssh_tunnel
+    def get_columns(self, table: str, database: str = None) -> list:
+        database = database or self.database
+        query = f"""
+            WITH table_info AS (
+                SELECT c.oid AS table_oid
+                FROM pg_class c
+                JOIN pg_namespace n ON c.relnamespace = n.oid
+                WHERE n.nspname = '{database}'
+                AND c.relname = '{table}'
+            )
+            SELECT
+                a.attname AS column_name,
+                t.typname AS data_type,
+                NOT a.attnotnull AS nullable,
+                pg_get_expr(ad.adbin, ad.adrelid) AS "default",
+                col_description(a.attrelid, a.attnum) AS comment
+            FROM pg_attribute a
+            JOIN table_info ti ON a.attrelid = ti.table_oid
+            JOIN pg_type t ON a.atttypid = t.oid
+            LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum
+            WHERE a.attnum > 0
+            AND NOT a.attisdropped
+            ORDER BY a.attnum;
+        """
+        result = self.fetchall(query)
+        column_metas = []
+        for row in result:
+            column_metas.append(
+                {
+                    "name": row[0],
+                    "type": row[1].lower() if row[1] else "",
+                    "nullable": row[2],
+                    "default": row[3],
+                    "comment": row[4],
+                }
+            )
+        return column_metas
+
+    @cached_property
+    @with_ssh_tunnel
+    def type_code_mapping(self) -> dict:
+        try:
+            rv = self.fetchall("SELECT oid, typname FROM pg_type")
+            return {row[0]: row[1] for row in rv if row[1]}
+        except Exception:
+            pass
+
+    def sqlalchemy_column_type_code_to_name(self, type_code: Any, size: int | None = None) -> str:
+        if self.type_code_mapping:
+            type_name = self.type_code_mapping.get(type_code)
+            if type_name:
+                return type_name
+
+        import psycopg2.extensions
+
+        pg_type = psycopg2.extensions.string_types.get(type_code)
+        pg_type_name = pg_type.name.lower() if pg_type else None
+        if pg_type_name in self.available_column_types:
+            return pg_type_name
+        return "varchar"
+
+    def convert_config_to_cube_config(
+        self, database: str, schema: str = None, datasource: DataSourceWrapper = None
+    ) -> dict:
+        return {
+            "type": "postgres",
+            "host": self.host,
+            "port": self.port,
+            "user": datasource.user,
+            "password": datasource.password,
+            "database": database or self.database,
+        }

recurvedata/connectors/connectors/python.py

@@ -0,0 +1,50 @@
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "python"
+UI_CONNECTION_TYPE = "Python"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class Python(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+
+    group = [ConnectorGroup.DESTINATION]
+    test_required = False
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "python_version": {
+                "type": "string",
+                "title": _l("Python Version"),
+                "enum": ["3.11.9", "3.10.14"],
+                "enumNames": ["3.11.9", "3.10.14"],
+                "default": "3.11.9",
+            },
+            "pyenv": {
+                "type": "string",
+                "title": _l("Python Virtual Environment Name"),
+                "default": "recurve_executor",
+            },
+            "requirements": {
+                "type": "string",
+                "title": _l("Python Package Requirements"),
+                "description": _l(
+                    "List of Python packages and versions to install, the same format as requirements.txt"
+                ),
+                "ui:options": {
+                    "type": "textarea",
+                    "rows": 10,
+                },
+            },
+        },
+        "order": ["python_version", "pyenv", "requirements"],
+        "required": ["python_version", "pyenv"],
+        "secret": [],
+    }
+
+    def test_connection(self):
+        pass

recurvedata/connectors/connectors/redshift.py

@@ -0,0 +1,187 @@
+from functools import cached_property
+from typing import Any
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.const import ENV_VAR_DBT_PASSWORD, ENV_VAR_DBT_USER
+from recurvedata.connectors.datasource import DataSourceWrapper
+from recurvedata.connectors.dbapi import DBAPIBase, with_ssh_tunnel
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "redshift"
+UI_CONNECTION_TYPE = "Redshift"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class RedshiftConnector(DBAPIBase):
+    SYSTEM_DATABASES = ["information_schema", "pg_catalog", "public", "temporary"]
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+    driver = "redshift+psycopg2"
+    setup_extras_require = ["psycopg2-binary", "sqlalchemy_redshift==0.8.15+recurve"]
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {"type": "string", "title": _l("Host Address")},
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 5439,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "database": {
+                "type": "string",
+                "title": _l("Database Name"),
+                "description": _l("The name of the database to connect to"),
+            },
+            "s3_options": {
+                "type": "object",
+                "title": _l("S3 Configuration"),
+                "description": _l("AWS S3 credentials for data loading and unloading"),
+                "properties": {
+                    "access_key_id": {"type": "string", "title": _l("AWS Access Key ID")},
+                    "secret_access_key": {"type": "string", "title": _l("AWS Secret Access Key")},
+                    "region": {"type": "string", "title": _l("AWS Region")},
+                },
+                "order": ["access_key_id", "secret_access_key", "region"],
+            },
+        },
+        "order": ["host", "port", "user", "password", "database", "s3_options"],
+        "required": ["host", "port"],
+        "secret": ["password"],
+    }
+
+    # All supported Redshift data types based on official documentation
+    available_column_types = [
+        # Numeric types
+        "smallint",
+        "int2",
+        "integer",
+        "int",
+        "int4",
+        "bigint",
+        "int8",
+        "decimal",
+        "numeric",
+        "real",
+        "float",
+        "float4",
+        "double precision",
+        "float8",
+        # Character types
+        "char",
+        "character",
+        "nchar",
+        "bpchar",
+        "varchar",
+        "character varying",
+        "nvarchar",
+        "text",
+        # Datetime types
+        "date",
+        "timestamp",
+        "timestamptz",
+        "timestamp with time zone",
+        "timestamp without time zone",
+        "time",
+        "timetz",
+        "time with time zone",
+        "time without time zone",
+        # Boolean type
+        "boolean",
+        "bool",
+        # Special types
+        "super",
+        "hllsketch",
+        "geometry",
+        "geography",
+        "varbyte",
+    ]
+
+    def convert_config_to_dbt_profile(self, database: str, schema: str = None) -> dict:
+        return {
+            "host": self.host,
+            "port": self.port,
+            "user": ENV_VAR_DBT_USER,
+            "password": ENV_VAR_DBT_PASSWORD,
+            "dbname": database,
+            "schema": schema,
+            "type": self.connection_type,
+            "threads": 10,
+        }
+
+    @with_ssh_tunnel
+    def get_columns(self, table: str, database=None):
+        database = database or self.database
+        query = f"""
+            WITH table_info AS (
+                SELECT c.oid AS table_oid
+                FROM pg_class c
+                JOIN pg_namespace n ON c.relnamespace = n.oid
+                WHERE n.nspname = '{database}'
+                AND c.relname = '{table}'
+            )
+            SELECT
+                a.attname AS column_name,
+                t.typname AS data_type,
+                NOT a.attnotnull AS nullable,
+                pg_get_expr(ad.adbin, ad.adrelid) AS "default",
+                col_description(a.attrelid, a.attnum) AS comment
+            FROM pg_attribute a
+            JOIN table_info ti ON a.attrelid = ti.table_oid
+            JOIN pg_type t ON a.atttypid = t.oid
+            LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum
+            WHERE a.attnum > 0
+            AND NOT a.attisdropped
+            ORDER BY a.attnum;
+        """
+        result = self.fetchall(query)
+        column_metas = []
+        for row in result:
+            column_metas.append(
+                {
+                    "name": row[0],
+                    "type": row[1].lower() if row[1] else "",
+                    "nullable": row[2],
+                    "default": row[3],
+                    "comment": row[4],
+                }
+            )
+        return column_metas
+
+    @cached_property
+    @with_ssh_tunnel
+    def type_code_mapping(self) -> dict:
+        try:
+            rv = self.fetchall("SELECT oid, typname FROM pg_catalog.pg_type")
+            return {row[0]: row[1] for row in rv if row[1]}
+        except Exception:
+            pass
+
+    def sqlalchemy_column_type_code_to_name(self, type_code: Any, size: int | None = None) -> str:
+        if self.type_code_mapping:
+            type_name = self.type_code_mapping.get(type_code)
+            if type_name:
+                return type_name
+
+        import psycopg2.extensions
+
+        pg_type = psycopg2.extensions.string_types.get(type_code)
+        pg_type_name = pg_type.name.lower() if pg_type else None
+        if pg_type_name in self.available_column_types:
+            return pg_type_name
+        return "varchar"
+
+    def convert_config_to_cube_config(
+        self, database: str, schema: str = None, datasource: DataSourceWrapper = None
+    ) -> dict:
+        return {
+            "type": "redshift",
+            "host": self.host,
+            "port": self.port,
+            "user": datasource.user,
+            "password": datasource.password,
+            "database": database or self.database,
+        }

recurvedata/connectors/connectors/s3.py

@@ -0,0 +1,93 @@
+import logging
+import os
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.object_store import ObjectStoreMixin
+from recurvedata.connectors.proxy import HTTP_PROXY_CONFIG_SCHEMA, HttpProxyMixin
+from recurvedata.connectors.utils import juice_sync_process_special_character_within_secret
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+try:
+    import boto3
+    from botocore.config import Config
+    from s3fs import S3FileSystem
+except ImportError:
+    S3FileSystem = None
+
+CONNECTION_TYPE = "s3"
+UI_CONNECTION_TYPE = "AWS S3"
+
+logger = logging.getLogger(__name__)
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class S3(HttpProxyMixin, ObjectStoreMixin):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+    setup_extras_require = ["fsspec[s3]", "s3fs>=2021.08", "boto3"]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "access_key_id": {"type": "string", "title": _l("AWS Access Key ID")},
+            "secret_access_key": {"type": "string", "title": _l("AWS Secret Access Key")},
+            "region": {"type": "string", "title": _l("AWS Region")},
+            "bucket": {"type": "string", "title": _l("Bucket Name")},
+            "proxies": HTTP_PROXY_CONFIG_SCHEMA["proxies"],
+        },
+        "order": ["access_key_id", "secret_access_key", "region", "bucket", "proxies"],
+        "required": ["access_key_id", "secret_access_key"],
+        "secret": ["secret_access_key"],
+    }
+
+    def init_connection(self, conf) -> S3FileSystem:
+        # todo: proxy
+        with self._init_proxy_manager():
+            client_kwargs = {}
+            if "region" in conf and conf["region"]:
+                client_kwargs["region_name"] = conf["region"]
+            con = S3FileSystem(key=conf["access_key_id"], secret=conf["secret_access_key"], client_kwargs=client_kwargs)
+            self.connector = con
+            return con
+
+    def test_connection(self):
+        with self._init_proxy_manager():
+            logger.info(
+                f'test s3 connection with bucket {self.bucket} and region {self.region}, proxy: {os.environ.get("http_proxy")}'
+            )
+
+            session = boto3.Session(
+                aws_access_key_id=self.access_key_id,
+                aws_secret_access_key=self.secret_access_key,
+                region_name=self.region,
+            )
+            timeout = 30
+            s3_client = session.client("s3", config=Config(connect_timeout=timeout, read_timeout=timeout))
+            if self.bucket:
+                s3_client.list_objects_v2(Bucket=self.bucket, MaxKeys=1)
+            else:
+                s3_client.list_buckets()
+
+    @property
+    def endpoint(self):
+        if not self.region:
+            raise ValueError("there is no region for endpoint")
+        return f"s3.{self.region}.amazonaws.com"
+
+    juice_sync_able = True
+
+    def juice_sync_path(self, path: str) -> str:
+        """
+        :param path: the s3 path, note that path is not started with bucket
+        :return:
+        """
+        if not self.bucket:
+            raise ValueError("the connection bucket cannot be empty in juice sync")
+        secret_part = f"{self.access_key_id}:{self.secret_access_key}"
+        secret_part = juice_sync_process_special_character_within_secret(secret_part)
+        endpoint_path_part = f'{self.bucket}.{self.endpoint}/{path.lstrip("/")}'
+        path_with_secret = f"s3://{secret_part}@{endpoint_path_part}"
+        path_without_secret = f"s3://{endpoint_path_part}"
+        return path_with_secret, path_without_secret

recurvedata/connectors/connectors/sftp.py

@@ -0,0 +1,87 @@
+import os
+
+from recurvedata.consts import ConnectorGroup
+
+try:
+    from fsspec.implementations.sftp import SFTPFileSystem
+except ImportError:
+    SFTPFileSystem = None
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.ftp import FTPMixin
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "sftp"
+UI_CONNECTION_TYPE = "SFTP"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class SFTP(FTPMixin):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+    setup_extras_require = ["fsspec[sftp]", "paramiko"]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "host": {
+                "type": "string",
+                "title": _l("Host Address"),
+                "default": "127.0.0.1",
+            },
+            "port": {
+                "type": "number",
+                "title": _l("Port Number"),
+                "default": 22,
+            },
+            "user": {"type": "string", "title": _l("Username")},
+            "password": {"type": "string", "title": _l("Password")},
+            "private_key_path": {"type": "string", "title": _l("Private Key File Path")},
+        },
+        "order": ["host", "port", "user", "password", "private_key_path"],
+        "required": ["host", "port"],
+        "secret": ["password"],
+    }
+
+    def _build_ssh_kwargs(self) -> dict:
+        """
+        build fsspec ssh_kwargs
+        :return:
+        """
+        import paramiko
+
+        pkey = None
+        pk_path = self.conf.get("private_key_path")
+        if pk_path:
+            pk_path = os.path.expanduser(pk_path)
+            pkey = paramiko.RSAKey.from_private_key_file(pk_path, password=self.conf.get("password"))
+        return {
+            "username": self.conf["user"],
+            "password": self.conf.get("password"),
+            "port": self.conf["port"],
+            "pkey": pkey,
+        }
+
+    def init_connection(self, conf) -> SFTPFileSystem:
+        con = SFTPFileSystem(host=conf["host"], **self._build_ssh_kwargs())
+        self.connector = con
+        return con
+
+    def test_connection(self):
+        self.connector.ls(".")
+
+    juice_sync_able = True
+
+    def juice_sync_path(self, path: str) -> str:
+        from urllib.parse import quote
+
+        username = self.conf["user"]
+        password = self.conf["password"]
+        password = quote(password)
+        port = self.conf["port"]
+        host = self.conf["host"]
+        # tmp only allow password
+        secret_path = f"{username}:{password}@{host}:{port}{path}"
+        non_secret_path = f"{username}:********@{host}:{port}{path}"
+        return secret_path, non_secret_path