recurvedata-lib 0.1.492__py2.py3-none-any.whl → 0.1.496__py2.py3-none-any.whl

recurvedata/__version__.py

@@ -1 +1 @@
-__version__ = "0.1.492"
+__version__ = "0.1.496"

recurvedata/client/client.py

@@ -1,8 +1,6 @@
 import json
 import logging
-import os
 import time
-from pathlib import Path
 from typing import Any, TypeVar, overload
 
 import httpx
@@ -26,90 +24,13 @@ class Client:
             config = AgentConfig.load()
         self.set_config(config)
 
-    @property
-    def is_offline_mode(self) -> bool:
-        """Check if offline mode is enabled via environment variable"""
-        return os.environ.get("RECURVE_OFFLINE_MODE", "").lower() in ("true", "1")
-
-    @property
-    def offline_data_path(self) -> Path:
-        """Get the offline data directory path"""
-        offline_path = os.environ.get("RECURVE_OFFLINE_DATA_PATH", "offline_data")
-        return Path(offline_path)
-
     def set_config(self, config: AgentConfig):
         self._config = config
-        # Only create HTTP client if not in offline mode
-        if not self.is_offline_mode:
-            self._client = httpx.Client(
-                base_url=config.server_url,
-                timeout=config.request_timeout,
-                headers={"User-Agent": f"RecurveLib/{__version__}"},
-            )
-        else:
-            self._client = None
-
-    def _resolve_offline_file_path(self, path: str, **kwargs) -> Path:
-        """Convert API path to local file path with parameterized support"""
-        # Remove leading /api/ prefix: /api/executor/connection -> executor/connection
-        if path.startswith("/api/"):
-            clean_path = path[5:]  # Remove "/api/" prefix
-        else:
-            clean_path = path.lstrip("/")
-        
-        # Extract parameters from kwargs
-        params = kwargs.get("params", {})
-        
-        # CORE OPERATOR APIs with parameter-based file structure:
-        
-        # 1. get_connection() API - parameterized by project_id + connection_name
-        if clean_path == "executor/connection":
-            project_id = params.get("project_id", "0")
-            connection_name = params.get("name", "default")
-            return self.offline_data_path / "executor/connection" / str(project_id) / f"{connection_name}.json"
-            
-        # 2. get_py_conn_configs() API - parameterized by project_id + project_connection_name  
-        elif clean_path == "executor/python-conn-configs":
-            project_id = params.get("project_id", "0")
-            # Python configs use project_connection_name as the key (fallback to other param names for compatibility)
-            # Handle empty strings properly - treat them as None/missing
-            project_connection_name = (params.get("project_connection_name") or 
-                                     params.get("project_conn_name") or 
-                                     params.get("pyenv_name") or 
-                                     "default")
-            return self.offline_data_path / "executor/python-conn-configs" / str(project_id) / f"{project_connection_name}.json"
-        
-        # For any other APIs, raise error do not support offline mode
-        raise APIError(f"Offline mode: {path} is not supported")
-
-    def _read_offline_data(self, method: str, path: str, response_model_class: type[ResponseModelType] | None = None, **kwargs) -> Any:
-        """Read API response from local JSON file"""
-        file_path = self._resolve_offline_file_path(path, **kwargs)
-        
-        logger.info(f"🔌 Offline mode: Reading from {file_path}")
-        
-        try:
-            if not file_path.exists():
-                logger.error(f"Offline data file not found: {file_path}")
-                raise APIError(f"Offline mode: Required data file not found: {file_path}")
-            
-            with open(file_path, 'r') as f:
-                resp_content = json.load(f)
-            
-            # Handle response model validation (same logic as online mode)
-            if response_model_class is not None:
-                if "code" in resp_content:
-                    return response_model_class.model_validate(resp_content["data"])
-                return response_model_class.model_validate(resp_content)
-            
-            return resp_content.get("data", resp_content)
-            
-        except APIError:
-            raise  # Re-raise APIError as-is
-        except Exception as e:
-            logger.error(f"Error reading offline data from {file_path}: {e}")
-            raise APIError(f"Offline mode: Failed to read data file {file_path}: {e}")
-
+        self._client = httpx.Client(
+            base_url=config.server_url,
+            timeout=config.request_timeout,
+            headers={"User-Agent": f"RecurveLib/{__version__}"},
+        )
 
     @overload
     def request(self, method: str, path: str, response_model_class: None = None, retries: int = 3, **kwargs) -> Any:
@@ -135,11 +56,6 @@ class Client:
         retries: int = 1,
         **kwargs,
     ) -> Any:
-        # Route to offline mode if enabled
-        if self.is_offline_mode:
-            return self._read_offline_data(method, path, response_model_class, **kwargs)
-        
-        # Original online mode logic
         self.prepare_header(kwargs)
         pre_err: httpx.HTTPStatusError | None = None
         for attempt in range(retries):
@@ -227,11 +143,8 @@ class Client:
                 )
 
     def close(self):
-        if self._client:
-            self._client.close()
+        self._client.close()
 
     @property
     def base_url(self) -> str:
-        if self.is_offline_mode:
-            return "offline://localhost"
-        return str(self._client.base_url) if self._client else ""
+        return str(self._client.base_url)

recurvedata/connectors/config_schema.py

@@ -1570,6 +1570,26 @@ ALL_CONFIG_SCHEMA_DCT = {
         "group": ["destination"],
         "test_required": True,
     },
+    "recurvedata.connectors.connectors.wecom": {
+        "type": "wecom",
+        "ui_type": "WeCom",
+        "category": ["others"],
+        "config_schema": {
+            "type": "object",
+            "properties": {
+                "webhook_url": {"type": "string", "title": _l("WeCom Webhook URL")},
+            },
+            "order": [
+                "webhook_url",
+            ],
+            "required": [
+                "webhook_url",
+            ],
+        },
+        "enabled": True,
+        "group": ["integration"],
+        "test_required": True,
+    },
 }
 
 # auto generated finish

recurvedata/connectors/connectors/wecom.py

@@ -0,0 +1,66 @@
+from urllib.parse import urlparse
+
+import requests
+
+from recurvedata.connectors._register import register_connector_class
+from recurvedata.connectors.base import RecurveConnectorBase
+from recurvedata.consts import ConnectorGroup
+from recurvedata.core.translation import _l
+
+CONNECTION_TYPE = "wecom"
+UI_CONNECTION_TYPE = "WeCom"
+
+
+@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
+class WeCom(RecurveConnectorBase):
+    connection_type = CONNECTION_TYPE
+    ui_connection_type = UI_CONNECTION_TYPE
+    group = [ConnectorGroup.INTEGRATION]
+
+    config_schema = {
+        "type": "object",
+        "properties": {
+            "webhook_url": {"type": "string", "title": _l("WeCom Webhook URL")},
+        },
+        "order": [
+            "webhook_url",
+        ],
+        "required": [
+            "webhook_url",
+        ],
+    }
+
+    @property
+    def webhook_url(self):
+        return self.conf["webhook_url"]
+
+    @property
+    def wecom_conf(self):
+        if hasattr(self, "_wecom_conf"):
+            return self._wecom_conf
+        self.init_config()
+        return self._wecom_conf
+
+    def test_connection(self):
+        """
+        Test WeCom webhook connection.
+        Validate URL format and network connectivity without sending actual messages.
+        """
+        # Validate URL format
+        parsed_url = urlparse(self.webhook_url)
+        if not parsed_url.scheme or not parsed_url.netloc:
+            raise ValueError("Invalid webhook URL format")
+
+        if parsed_url.scheme != "https":
+            raise ValueError("WeCom webhook URL must use HTTPS")
+
+        # Verify WeCom domain
+        if "qyapi.weixin.qq.com" not in parsed_url.netloc:
+            raise ValueError("Invalid WeCom webhook URL domain")
+
+        # Test network connectivity (use HEAD request without sending message body)
+        try:
+            requests.head(self.webhook_url, timeout=10, allow_redirects=True)
+            # Connection test only, status code not checked (HEAD may return 405)
+        except requests.exceptions.RequestException as e:
+            raise ValueError(f"Failed to connect to WeCom webhook: {str(e)}")

recurvedata/connectors/dbapi.py

@@ -397,19 +397,27 @@ class DBAPIBase(RecurveConnectorBase):
         set_env_dbt_password(self.password or "")
 
     @classmethod
-    def order_sql(cls, sql: str, orders: list[dict[str, str]] = None, return_sql: bool = True):
-        """
-        order the sql by the orders
-        """
+    def get_dialect(cls):
         # dialect impala -> hive, cuz there is no dialect 'impala' in sqlglot
-        dialect = "hive" if cls.connection_type == "impala" else (cls.connection_type or None)
+        return "hive" if cls.connection_type == "impala" else (cls.connection_type or None)
+
+    @classmethod
+    def clean_sql(cls, sql):
+        dialect = cls.get_dialect()
         # Parse the SQL query
         parsed = sqlglot.parse_one(sql, read=dialect)
         # since some sql dialects have special identifier, we need to use the dialect to generate the clean sql
-        clean_sql = parsed.sql(dialect=dialect, comments=False)
+        return parsed.sql(dialect=dialect, comments=False)
+
+    @classmethod
+    def order_sql(cls, sql: str, orders: list[dict[str, str]] = None, return_sql: bool = True):
+        """
+        order the sql by the orders
+        """
+        dialect = cls.get_dialect()
         # Wrap the entire query with a subquery
         alias = "_recurve_limit_subquery"
-        subquery = exp.Subquery(this=clean_sql, alias=alias)
+        subquery = exp.Subquery(this=cls.clean_sql(sql), alias=alias)
 
         # Create a new SELECT statement with the subquery and the LIMIT clause
         outer_select = exp.select("*").from_(subquery)
@@ -436,7 +444,7 @@ class DBAPIBase(RecurveConnectorBase):
         no validation on sql.
         If the sql is DML, then execute it will raise an error.
         """
-        dialect = "hive" if cls.connection_type == "impala" else (cls.connection_type or None)
+        dialect = cls.get_dialect()
 
         outer_select = cls.order_sql(sql, orders, return_sql=False)
 
@@ -456,7 +464,7 @@ class DBAPIBase(RecurveConnectorBase):
         no validation on sql.
         If the sql is DML, then execute it will raise an error.
         """
-        return f"SELECT COUNT(1) FROM ({sql}) AS cnt_subquery"
+        return f"SELECT COUNT(1) FROM ({cls.clean_sql(sql)}) AS cnt_subquery"
 
 
 @dataclass

recurvedata/schedulers/airflow_db_process.py

@@ -148,12 +148,15 @@ class AirflowDbService:
         max_execution_date: datetime.datetime | None,
         failed_only: bool,
     ):
-        drs: list[DagRun] = DagRun.find(
-            dag_id=dag.dag_id,
-            run_id=run_id,
-            execution_start_date=min_execution_date,
-            execution_end_date=max_execution_date,
-        )
+        kwargs = {
+            "dag_id": dag.dag_id,
+            "run_id": run_id,
+            "execution_start_date": min_execution_date,
+            "execution_end_date": max_execution_date,
+        }
+        if min_execution_date or max_execution_date:
+            kwargs.pop("run_id")
+        drs: list[DagRun] = DagRun.find(**kwargs)
         if not drs:
             logger.info(f"skip rerun, no dag_run found for {dag.dag_id} at {run_id}")
             return
@@ -182,14 +185,19 @@ class AirflowDbService:
         include_downstream: bool,
         failed_only: bool,
     ):
-        drs: list[DagRun] = DagRun.find(
-            dag_id=dag.dag_id,
-            run_id=run_id,
-            execution_start_date=min_execution_date,
-            execution_end_date=max_execution_date,
-        )
+        kwargs = {
+            "dag_id": dag.dag_id,
+            "run_id": run_id,
+            "execution_start_date": min_execution_date,
+            "execution_end_date": max_execution_date,
+        }
+        if min_execution_date or max_execution_date:
+            kwargs.pop("run_id")
+        drs: list[DagRun] = DagRun.find(**kwargs)
         if not drs:
-            logger.info(f"skip rerun, no dag_run found for {dag.dag_id} at {run_id}")
+            logger.info(
+                f"skip rerun, no dag_run found for {dag.dag_id} at {run_id}, or between {min_execution_date} and {max_execution_date}"
+            )
             return
         clear_start_date = min([dr.execution_date for dr in drs])
         clear_end_date = max([dr.execution_date for dr in drs])
@@ -205,14 +213,16 @@ class AirflowDbService:
                 for task_id in clear_task_ids:
                     if task_id in dag.task_dict:
                         task = dag.task_dict[task_id]
-                        upstream_task_ids = [t.task_id for t in task.upstream_list]
+                        # Use Airflow's built-in method to get all upstream tasks
+                        upstream_task_ids = task.get_flat_relative_ids(upstream=True)
                         expanded_task_ids.update(upstream_task_ids)
 
             if include_downstream:
                 for task_id in clear_task_ids:
                     if task_id in dag.task_dict:
                         task = dag.task_dict[task_id]
-                        downstream_task_ids = [t.task_id for t in task.downstream_list]
+                        # Use Airflow's built-in method to get all downstream tasks
+                        downstream_task_ids = task.get_flat_relative_ids(upstream=False)
                         expanded_task_ids.update(downstream_task_ids)
 
         clear_task_ids = list(expanded_task_ids)

{recurvedata_lib-0.1.492.dist-info → recurvedata_lib-0.1.496.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recurvedata-lib
-Version: 0.1.492
+Version: 0.1.496
 Summary: Common Library for ReOrc Data Platform
 Author-email: Reorc Team <contact@recurvedata.com>
 Requires-Dist: croniter
@@ -77,6 +77,7 @@ Requires-Dist: dbt-duckdb==1.8.4; extra == 'dbt'
 Requires-Dist: dbt-extractor==0.5.1; extra == 'dbt'
 Requires-Dist: dbt-fabric==1.9.4; extra == 'dbt'
 Requires-Dist: dbt-impala==1.8.0; extra == 'dbt'
+Requires-Dist: dbt-mysql==1.8.0+recurve.1; extra == 'dbt'
 Requires-Dist: dbt-postgres==1.8.2; extra == 'dbt'
 Requires-Dist: dbt-redshift==1.8.1; extra == 'dbt'
 Requires-Dist: dbt-semantic-interfaces==0.5.1; extra == 'dbt'

{recurvedata_lib-0.1.492.dist-info → recurvedata_lib-0.1.496.dist-info}/RECORD

@@ -1,5 +1,5 @@
 recurvedata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-recurvedata/__version__.py,sha256=nLCZ9XH8tY-NsC6qhMGzxaKDlG3iuUYkf6rh-HS8yLo,24
+recurvedata/__version__.py,sha256=Y8Rd0ubG5uL_xudTnGRSLi5Dxc73nGvHR1VMlDgyh0Y,24
 recurvedata/config.py,sha256=rbpccM6qr8ekdEC5p7XtsivayxmL64-Nb9ogrcWMgX8,3848
 recurvedata/consts.py,sha256=y5BuAHBrz1jAcS5NgZxnrkfomQv3_5hvgafYwpLKpV8,1224
 recurvedata/error_codes.py,sha256=y4OLrs0_2iLWdvQJEV10m-414uPkUdm4v0D7bE8iWOM,2303
@@ -7,15 +7,15 @@ recurvedata/exceptions.py,sha256=-Jtm1MXk06ViNOP176MRELFOujjYkZI_IkZY4hzwhRo,187
 recurvedata/provider_manager.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 recurvedata/schema.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 recurvedata/client/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
-recurvedata/client/client.py,sha256=UPbHZxb8q6l_pnwEZaD8nEFj_HMejBlhiLlWZAEqE3U,9806
+recurvedata/client/client.py,sha256=Z3JccGzQbRoWr0qnfvUam2_7zxOnyFSn1wV5A5F6jUI,5613
 recurvedata/client/server_client.py,sha256=bZ55S_tk_fI3JDLU3txha2HKbS4hKUG6jLehj3HnQc0,3033
 recurvedata/connectors/__init__.py,sha256=1VpGyGu9FA7lAvKZv0Z8j9ZzSi4i-L3_PyLcxdbrfs4,577
 recurvedata/connectors/_register.py,sha256=7NYVIJk9PufhTJWyj7JkPt9p66Cc1ieCBPpFi24cMwo,1639
 recurvedata/connectors/base.py,sha256=tuCUq2hij8GknWUKs1sP8HC_JZQie2csxoy3s-7Bb_4,3714
-recurvedata/connectors/config_schema.py,sha256=iGiZ-uKwMHM4LGlrlHAVdIpHoJ148wDa1L53HHKSH6k,62286
+recurvedata/connectors/config_schema.py,sha256=fcTSyeVRA3tKvhD_kAvjk9lKo9TZlbDYRrIw9DKLkfk,62854
 recurvedata/connectors/const.py,sha256=3Zl4wZ0AP6t9x63QoRM6GMG6ZDgTQW-JacOv7lJBcwI,10153
 recurvedata/connectors/datasource.py,sha256=w09SskSeNJjNc7qUEp0N3UV-YnMX2flzTutg1BaLqO0,5573
-recurvedata/connectors/dbapi.py,sha256=tmmk7JdecI6NMu_HiLdxWeRpZjX2B4t1RY10SytwBNU,14173
+recurvedata/connectors/dbapi.py,sha256=I9KlaOdWwDrOtZeDAZdBU1MxmkARnLCkZDnY_Vm9llI,14293
 recurvedata/connectors/fs.py,sha256=ZZlDq4Bd7aA09ZarZwyWrzFEwUgG5GHm8JPctHw7Ako,1460
 recurvedata/connectors/ftp.py,sha256=9H_xON87ESpatU5e8kCwwbYwLRnIH1ClqazIbDe-YfU,1091
 recurvedata/connectors/object_store.py,sha256=Fd3jGnUQ79sNuC8L-0nSDxwHh32emU0RuIqTAOZx4IM,1714
@@ -60,6 +60,7 @@ recurvedata/connectors/connectors/spark.py,sha256=zaTrsA0fgIHLLR3LZ_D9pd3KIccZBe
 recurvedata/connectors/connectors/starrocks.py,sha256=IdrlcKz0vUASokdR4QVjZqCXoVZ_dFBEdJUnDlMaScQ,6028
 recurvedata/connectors/connectors/tencent_cos.py,sha256=1f_31aNW8lznuygE_N7tQbK9PToGIRUkFnmhHSRmq54,1392
 recurvedata/connectors/connectors/tidb.py,sha256=2VkZ8x6fhmbcGRc7ekTwOADGgWiRFVHr4tnzuQVfqDU,1924
+recurvedata/connectors/connectors/wecom.py,sha256=1n6DZeaSEEWDiLwuMWIxx1QYpOPJPKQUACUtGcgEvUQ,2134
 recurvedata/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 recurvedata/core/config.py,sha256=_wmsWpPNgZE-mUSn1u3C7m4Zf5j2XEhZLhtUY5llbYo,1436
 recurvedata/core/configurable.py,sha256=RekN9mY46Sb_IPLW5aciNg0xF8E47CtMbHPOsar3tfU,719
@@ -256,7 +257,7 @@ recurvedata/providers/flywheel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 recurvedata/providers/mysql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 recurvedata/schedulers/__init__.py,sha256=euiId3-BC5JpPmXb2gSzXBycfLMCJyvjn3c5D2ZqK8U,21
 recurvedata/schedulers/airflow.py,sha256=lX3nc-m2vrwuNuSABL560oaMR6t9G5mwhrKCExmrQDw,36198
-recurvedata/schedulers/airflow_db_process.py,sha256=_WxtWg5uFQrGV3JeXWWl2jxAZQ7bnXmvMCyXSCNqycw,13459
+recurvedata/schedulers/airflow_db_process.py,sha256=9Xm6SN6PFrrWMES5UHnsutCVNRg2Yq_BzhVjlFFAy0E,13968
 recurvedata/schedulers/airflow_operators.py,sha256=-AfLwT9Oi8_JDCclc2XhspZ13cbTcu6ftj6yBw2-AWw,2011
 recurvedata/schedulers/airflow_plugin.py,sha256=aUE1YxXJuMR4hL2hJ1XailBw3wA-V_ljYBVQz39MvYE,242
 recurvedata/schedulers/airflow_trigger_dag_patch.py,sha256=bUVYRYDozgYeLcpQEmLZsNKoFbVS2PvuSwnpbIyTS0U,4300
@@ -327,7 +328,7 @@ recurvedata/utils/singleton.py,sha256=15PaK2nP9H5PyO26IZzQPpfzlW5h_Bp1NHA6QPb4H0
 recurvedata/utils/sql.py,sha256=u3XRPv8_vsrMFMm-O1xyV63ZXChAFVHmJj2_xbRwcNg,264
 recurvedata/utils/timeout.py,sha256=U5ssSgoyVRqop9P8vmyI3BJI-OnMH2k22PdzTh-JN4c,780
 recurvedata/utils/tracing.py,sha256=gpK8q00ZjZmI81YpgQtDBPLzBvVSYpPA0sIq4wqnvBc,472
-recurvedata_lib-0.1.492.dist-info/METADATA,sha256=4t9YLZ1Kg0mYym7vIsUURwNgtxqTH5FXx-V2HlIMqLU,27743
-recurvedata_lib-0.1.492.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
-recurvedata_lib-0.1.492.dist-info/entry_points.txt,sha256=4KBBIfooz3wqXBoLlidRRP4_r36JUCnIF4BFn4igtms,209
-recurvedata_lib-0.1.492.dist-info/RECORD,,
+recurvedata_lib-0.1.496.dist-info/METADATA,sha256=BiSQLgBoteUjIHtU3IBWdDhKF_rr8CkQ7cB5rqu7avM,27801
+recurvedata_lib-0.1.496.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
+recurvedata_lib-0.1.496.dist-info/entry_points.txt,sha256=4KBBIfooz3wqXBoLlidRRP4_r36JUCnIF4BFn4igtms,209
+recurvedata_lib-0.1.496.dist-info/RECORD,,