recurvedata-lib 0.1.491__tar.gz → 0.1.496__tar.gz

{recurvedata_lib-0.1.491 → recurvedata_lib-0.1.496}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recurvedata-lib
-Version: 0.1.491
+Version: 0.1.496
 Summary: Common Library for ReOrc Data Platform
 Author-email: Reorc Team <contact@recurvedata.com>
 Requires-Dist: croniter
@@ -77,6 +77,7 @@ Requires-Dist: dbt-duckdb==1.8.4; extra == 'dbt'
 Requires-Dist: dbt-extractor==0.5.1; extra == 'dbt'
 Requires-Dist: dbt-fabric==1.9.4; extra == 'dbt'
 Requires-Dist: dbt-impala==1.8.0; extra == 'dbt'
+Requires-Dist: dbt-mysql==1.8.0+recurve.1; extra == 'dbt'
 Requires-Dist: dbt-postgres==1.8.2; extra == 'dbt'
 Requires-Dist: dbt-redshift==1.8.1; extra == 'dbt'
 Requires-Dist: dbt-semantic-interfaces==0.5.1; extra == 'dbt'

{recurvedata_lib-0.1.491 → recurvedata_lib-0.1.496}/pyproject.toml

@@ -363,6 +363,7 @@ dbt = [
     "dbt-redshift==1.8.1",
     "dbt-semantic-interfaces==0.5.1",
     "dbt-starrocks==1.7.0",
+    "dbt-mysql==1.8.0+recurve.1",
     "deepdiff==7.0.1",
     "deprecated==1.2.15",
     "distlib==0.3.9",

recurvedata_lib-0.1.496/recurvedata/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.496"

{recurvedata_lib-0.1.491 → recurvedata_lib-0.1.496}/recurvedata/connectors/config_schema.py

@@ -1073,6 +1073,84 @@ ALL_CONFIG_SCHEMA_DCT = {
         "group": ["destination"],
         "test_required": True,
     },
+    "recurvedata.connectors.connectors.n8n": {
+        "type": "n8n",
+        "ui_type": "n8n",
+        "category": ["others"],
+        "config_schema": {
+            "type": "object",
+            "properties": {
+                "url": {
+                    "type": "string",
+                    "title": _l("API Address"),
+                    "description": _l("The URL of the n8n API, e.g. https://localhost:5678/api/v1"),
+                },
+                "api_key": {"type": "string", "title": _l("API KEY")},
+                "timeout": {
+                    "type": "number",
+                    "title": _l("Timeout"),
+                    "description": _l("The timeout of the n8n API, e.g. 60"),
+                    "default": 60,
+                },
+                "webhook_credential": {
+                    "type": "object",
+                    "title": _l("Webhook Trigger Node Credential"),
+                    "description": _l("The credential of the n8n webhook trigger node"),
+                    "properties": {
+                        "credential_type": {
+                            "type": "string",
+                            "title": _l("Credential Type"),
+                            "enum": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
+                            "enumNames": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
+                            "default": "None",
+                        },
+                        "basic_auth": {
+                            "ui:hidden": '{{parentFormData.credential_type !== "Basic Auth"}}',
+                            "type": "object",
+                            "title": _l("Basic Auth"),
+                            "description": _l("The basic auth of the n8n webhook trigger node"),
+                            "properties": {
+                                "username": {"type": "string", "title": _l("Username")},
+                                "password": {"type": "string", "title": _l("Password")},
+                            },
+                        },
+                        "header_auth": {
+                            "ui:hidden": '{{parentFormData.credential_type !== "Header Auth"}}',
+                            "type": "object",
+                            "title": _l("Header Auth"),
+                            "description": _l("The header auth of the n8n webhook trigger node"),
+                            "properties": {
+                                "header_name": {"type": "string", "title": _l("Header Name")},
+                                "header_value": {"type": "string", "title": _l("Header Value")},
+                            },
+                        },
+                        "jwt_auth": {
+                            "ui:hidden": '{{parentFormData.credential_type !== "JWT Auth"}}',
+                            "type": "object",
+                            "title": _l("JWT Auth"),
+                            "description": _l("The jwt auth of the n8n webhook trigger node"),
+                            "properties": {
+                                "jwt_token": {"type": "string", "title": _l("JWT Token")},
+                            },
+                        },
+                    },
+                    "order": ["credential_type", "basic_auth", "header_auth", "jwt_auth"],
+                },
+                "proxies": HTTP_PROXY_CONFIG_SCHEMA["proxies"],
+            },
+            "order": ["url", "api_key", "timeout", "webhook_credential", "proxies"],
+            "required": ["url", "api_key"],
+            "secret": [
+                "api_key",
+                "webhook_credential.basic_auth.password",
+                "webhook_credential.header_auth.header_value",
+                "webhook_credential.jwt_auth.jwt_token",
+            ],
+        },
+        "enabled": True,
+        "group": ["destination"],
+        "test_required": True,
+    },
     "recurvedata.connectors.connectors.oss": {
         "type": "oss",
         "ui_type": "Aliyun OSS",
@@ -1492,6 +1570,26 @@ ALL_CONFIG_SCHEMA_DCT = {
         "group": ["destination"],
         "test_required": True,
     },
+    "recurvedata.connectors.connectors.wecom": {
+        "type": "wecom",
+        "ui_type": "WeCom",
+        "category": ["others"],
+        "config_schema": {
+            "type": "object",
+            "properties": {
+                "webhook_url": {"type": "string", "title": _l("WeCom Webhook URL")},
+            },
+            "order": [
+                "webhook_url",
+            ],
+            "required": [
+                "webhook_url",
+            ],
+        },
+        "enabled": True,
+        "group": ["integration"],
+        "test_required": True,
+    },
 }
 
 # auto generated finish

recurvedata_lib-0.1.496/recurvedata/connectors/connectors/n8n.py

@@ -0,0 +1,141 @@
+import base64
+from contextlib import contextmanager
+from urllib.parse import urlparse
+
+import httpx
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.connectors.proxy import HTTP_PROXY_CONFIG_SCHEMA, HttpProxyMixin
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "n8n"
+UI_CONNECTION_TYPE = "n8n"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class N8N(HttpProxyMixin, RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.DESTINATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "title": _l("API Address"),
+                "description": _l("The URL of the n8n API, e.g. https://localhost:5678/api/v1"),
+            },
+            "api_key": {"type": "string", "title": _l("API KEY")},
+            "timeout": {
+                "type": "number",
+                "title": _l("Timeout"),
+                "description": _l("The timeout of the n8n API, e.g. 60"),
+                "default": 60,
+            },
+            "webhook_credential": {
+                "type": "object",
+                "title": _l("Webhook Trigger Node Credential"),
+                "description": _l("The credential of the n8n webhook trigger node"),
+                "properties": {
+                    "credential_type": {
+                        "type": "string",
+                        "title": _l("Credential Type"),
+                        "enum": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
+                        "enumNames": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
+                        "default": "None",
+                    },
+                    "basic_auth": {
+                        "ui:hidden": '{{parentFormData.credential_type !== "Basic Auth"}}',
+                        "type": "object",
+                        "title": _l("Basic Auth"),
+                        "description": _l("The basic auth of the n8n webhook trigger node"),
+                        "properties": {
+                            "username": {"type": "string", "title": _l("Username")},
+                            "password": {"type": "string", "title": _l("Password")},
+                        },
+                    },
+                    "header_auth": {
+                        "ui:hidden": '{{parentFormData.credential_type !== "Header Auth"}}',
+                        "type": "object",
+                        "title": _l("Header Auth"),
+                        "description": _l("The header auth of the n8n webhook trigger node"),
+                        "properties": {
+                            "header_name": {"type": "string", "title": _l("Header Name")},
+                            "header_value": {"type": "string", "title": _l("Header Value")},
+                        },
+                    },
+                    "jwt_auth": {
+                        "ui:hidden": '{{parentFormData.credential_type !== "JWT Auth"}}',
+                        "type": "object",
+                        "title": _l("JWT Auth"),
+                        "description": _l("The jwt auth of the n8n webhook trigger node"),
+                        "properties": {
+                            "jwt_token": {"type": "string", "title": _l("JWT Token")},
+                        },
+                    },
+                },
+                "order": ["credential_type", "basic_auth", "header_auth", "jwt_auth"],
+            },
+            "proxies": HTTP_PROXY_CONFIG_SCHEMA["proxies"],
+        },
+        "order": ["url", "api_key", "timeout", "webhook_credential", "proxies"],
+        "required": ["url", "api_key"],
+        "secret": [
+            "api_key",
+            "webhook_credential.basic_auth.password",
+            "webhook_credential.header_auth.header_value",
+            "webhook_credential.jwt_auth.jwt_token",
+        ],
+    }
+
+    def test_connection(self):
+        pass
+
+    @contextmanager
+    def _n8n_client(self) -> httpx.Client:
+        with self._init_proxy_manager():
+            yield httpx.Client(
+                base_url=f"{self.url}", headers={"X-N8N-API-KEY": f"{self.api_key}"}, timeout=self.timeout
+            )
+
+    def get_workflows(self) -> list[dict]:
+        path = "/workflows"
+        workflows = []
+        cursor = None
+        with self._n8n_client() as client:
+            response = client.get(path)
+            workflows.extend(response.json()["data"])
+            if response.json()["nextCursor"] and response.json()["nextCursor"] != cursor:
+                cursor = response.json()["nextCursor"]
+                while cursor:
+                    response = client.get(path, params={"cursor": cursor})
+                    workflows.extend(response.json()["data"])
+                    cursor = response.json()["nextCursor"]
+            return workflows
+
+    def _trigger_workflow_via_webhook(self, webhook_id: str, payload: dict) -> dict:
+        main_url = f"{urlparse(self.url).scheme}://{urlparse(self.url).netloc}"
+        webhook_url = f"{main_url}/webhook/{webhook_id}"
+        headers = {}
+        credential_type = self.webhook_credential.get("credential_type")
+        basic_auth = self.webhook_credential.get("basic_auth", {})
+        header_auth = self.webhook_credential.get("header_auth", {})
+        jwt_auth = self.webhook_credential.get("jwt_auth", {})
+        with self._init_proxy_manager():
+            if credential_type == "Basic Auth":
+                username = basic_auth.get("username", "")
+                password = basic_auth.get("password", "")
+                headers["Authorization"] = f'Basic {base64.b64encode(f"{username}:{password}".encode()).decode()}'
+            elif credential_type == "Header Auth":
+                header_name = header_auth.get("header_name", "")
+                header_value = header_auth.get("header_value", "")
+                headers[header_name] = header_value
+            elif credential_type == "JWT Auth":
+                jwt_token = jwt_auth.get("jwt_token", "")
+                headers["Authorization"] = f"Bearer {jwt_token}"
+            response = httpx.post(url=webhook_url, headers=headers, timeout=self.timeout, json=payload)
+
+            return response.json()

recurvedata_lib-0.1.496/recurvedata/connectors/connectors/wecom.py

@@ -0,0 +1,66 @@
+from urllib.parse import urlparse
+
+import requests
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "wecom"
+UI_CONNECTION_TYPE = "WeCom"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class WeCom(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.INTEGRATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "webhook_url": {"type": "string", "title": _l("WeCom Webhook URL")},
+        },
+        "order": [
+            "webhook_url",
+        ],
+        "required": [
+            "webhook_url",
+        ],
+    }
+
+    @property
+    def webhook_url(self):
+        return self.conf["webhook_url"]
+
+    @property
+    def wecom_conf(self):
+        if hasattr(self, "_wecom_conf"):
+            return self._wecom_conf
+        self.init_config()
+        return self._wecom_conf
+
+    def test_connection(self):
+        """
+        Test WeCom webhook connection.
+        Validate URL format and network connectivity without sending actual messages.
+        """
+        # Validate URL format
+        parsed_url = urlparse(self.webhook_url)
+        if not parsed_url.scheme or not parsed_url.netloc:
+            raise ValueError("Invalid webhook URL format")
+
+        if parsed_url.scheme != "https":
+            raise ValueError("WeCom webhook URL must use HTTPS")
+
+        # Verify WeCom domain
+        if "qyapi.weixin.qq.com" not in parsed_url.netloc:
+            raise ValueError("Invalid WeCom webhook URL domain")
+
+        # Test network connectivity (use HEAD request without sending message body)
+        try:
+            requests.head(self.webhook_url, timeout=10, allow_redirects=True)
+            # Connection test only, status code not checked (HEAD may return 405)
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to connect to WeCom webhook: {str(e)}")

{recurvedata_lib-0.1.491 → recurvedata_lib-0.1.496}/recurvedata/connectors/dbapi.py

@@ -397,19 +397,27 @@ class DBAPIBase(RecurveConnectorBase):
         set_env_dbt_password(self.password or "")
 
     @classmethod
-    def order_sql(cls, sql: str, orders: list[dict[str, str]] = None, return_sql: bool = True):
-        """
-        order the sql by the orders
-        """
+    def get_dialect(cls):
         # dialect impala -> hive, cuz there is no dialect 'impala' in sqlglot
-        dialect = "hive" if cls.connection_type == "impala" else (cls.connection_type or None)
+        return "hive" if cls.connection_type == "impala" else (cls.connection_type or None)
+
+    @classmethod
+    def clean_sql(cls, sql):
+        dialect = cls.get_dialect()
         # Parse the SQL query
         parsed = sqlglot.parse_one(sql, read=dialect)
         # since some sql dialects have special identifier, we need to use the dialect to generate the clean sql
-        clean_sql = parsed.sql(dialect=dialect, comments=False)
+        return parsed.sql(dialect=dialect, comments=False)
+
+    @classmethod
+    def order_sql(cls, sql: str, orders: list[dict[str, str]] = None, return_sql: bool = True):
+        """
+        order the sql by the orders
+        """
+        dialect = cls.get_dialect()
         # Wrap the entire query with a subquery
         alias = "_recurve_limit_subquery"
-        subquery = exp.Subquery(this=clean_sql, alias=alias)
+        subquery = exp.Subquery(this=cls.clean_sql(sql), alias=alias)
 
         # Create a new SELECT statement with the subquery and the LIMIT clause
         outer_select = exp.select("*").from_(subquery)
@@ -436,7 +444,7 @@ class DBAPIBase(RecurveConnectorBase):
         no validation on sql.
         If the sql is DML, then execute it will raise an error.
         """
-        dialect = "hive" if cls.connection_type == "impala" else (cls.connection_type or None)
+        dialect = cls.get_dialect()
 
         outer_select = cls.order_sql(sql, orders, return_sql=False)
 
@@ -456,7 +464,7 @@ class DBAPIBase(RecurveConnectorBase):
         no validation on sql.
         If the sql is DML, then execute it will raise an error.
         """
-        return f"SELECT COUNT(1) FROM ({sql}) AS cnt_subquery"
+        return f"SELECT COUNT(1) FROM ({cls.clean_sql(sql)}) AS cnt_subquery"
 
 
 @dataclass

recurvedata_lib-0.1.491/recurvedata/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.1.491"