recurvedata-lib 0.1.488__tar.gz → 0.1.490__tar.gz

{recurvedata_lib-0.1.488 → recurvedata_lib-0.1.490}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: recurvedata-lib
-Version: 0.1.488
+Version: 0.1.490
 Summary: Common Library for ReOrc Data Platform
 Author-email: Reorc Team <contact@recurvedata.com>
 Requires-Dist: croniter

recurvedata_lib-0.1.490/recurvedata/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.1.490"

{recurvedata_lib-0.1.488 → recurvedata_lib-0.1.490}/recurvedata/client/client.py

@@ -1,8 +1,6 @@
 import json
 import logging
-import os
 import time
-from pathlib import Path
 from typing import Any, TypeVar, overload
 
 import httpx
@@ -26,90 +24,13 @@ class Client:
             config = AgentConfig.load()
         self.set_config(config)
 
-    @property
-    def is_offline_mode(self) -> bool:
-        """Check if offline mode is enabled via environment variable"""
-        return os.environ.get("RECURVE_OFFLINE_MODE", "").lower() in ("true", "1")
-
-    @property
-    def offline_data_path(self) -> Path:
-        """Get the offline data directory path"""
-        offline_path = os.environ.get("RECURVE_OFFLINE_DATA_PATH", "offline_data")
-        return Path(offline_path)
-
     def set_config(self, config: AgentConfig):
         self._config = config
-        # Only create HTTP client if not in offline mode
-        if not self.is_offline_mode:
-            self._client = httpx.Client(
-                base_url=config.server_url,
-                timeout=config.request_timeout,
-                headers={"User-Agent": f"RecurveLib/{__version__}"},
-            )
-        else:
-            self._client = None
-
-    def _resolve_offline_file_path(self, path: str, **kwargs) -> Path:
-        """Convert API path to local file path with parameterized support"""
-        # Remove leading /api/ prefix: /api/executor/connection -> executor/connection
-        if path.startswith("/api/"):
-            clean_path = path[5:]  # Remove "/api/" prefix
-        else:
-            clean_path = path.lstrip("/")
-        
-        # Extract parameters from kwargs
-        params = kwargs.get("params", {})
-        
-        # CORE OPERATOR APIs with parameter-based file structure:
-        
-        # 1. get_connection() API - parameterized by project_id + connection_name
-        if clean_path == "executor/connection":
-            project_id = params.get("project_id", "0")
-            connection_name = params.get("name", "default")
-            return self.offline_data_path / "executor/connection" / str(project_id) / f"{connection_name}.json"
-            
-        # 2. get_py_conn_configs() API - parameterized by project_id + project_connection_name  
-        elif clean_path == "executor/python-conn-configs":
-            project_id = params.get("project_id", "0")
-            # Python configs use project_connection_name as the key (fallback to other param names for compatibility)
-            # Handle empty strings properly - treat them as None/missing
-            project_connection_name = (params.get("project_connection_name") or 
-                                     params.get("project_conn_name") or 
-                                     params.get("pyenv_name") or 
-                                     "default")
-            return self.offline_data_path / "executor/python-conn-configs" / str(project_id) / f"{project_connection_name}.json"
-        
-        # For any other APIs, raise error do not support offline mode
-        raise APIError(f"Offline mode: {path} is not supported")
-
-    def _read_offline_data(self, method: str, path: str, response_model_class: type[ResponseModelType] | None = None, **kwargs) -> Any:
-        """Read API response from local JSON file"""
-        file_path = self._resolve_offline_file_path(path, **kwargs)
-        
-        logger.info(f"🔌 Offline mode: Reading from {file_path}")
-        
-        try:
-            if not file_path.exists():
-                logger.error(f"Offline data file not found: {file_path}")
-                raise APIError(f"Offline mode: Required data file not found: {file_path}")
-            
-            with open(file_path, 'r') as f:
-                resp_content = json.load(f)
-            
-            # Handle response model validation (same logic as online mode)
-            if response_model_class is not None:
-                if "code" in resp_content:
-                    return response_model_class.model_validate(resp_content["data"])
-                return response_model_class.model_validate(resp_content)
-            
-            return resp_content.get("data", resp_content)
-            
-        except APIError:
-            raise  # Re-raise APIError as-is
-        except Exception as e:
-            logger.error(f"Error reading offline data from {file_path}: {e}")
-            raise APIError(f"Offline mode: Failed to read data file {file_path}: {e}")
-
+        self._client = httpx.Client(
+            base_url=config.server_url,
+            timeout=config.request_timeout,
+            headers={"User-Agent": f"RecurveLib/{__version__}"},
+        )
 
     @overload
     def request(self, method: str, path: str, response_model_class: None = None, retries: int = 3, **kwargs) -> Any:
@@ -135,11 +56,6 @@ class Client:
         retries: int = 1,
         **kwargs,
     ) -> Any:
-        # Route to offline mode if enabled
-        if self.is_offline_mode:
-            return self._read_offline_data(method, path, response_model_class, **kwargs)
-        
-        # Original online mode logic
         self.prepare_header(kwargs)
         pre_err: httpx.HTTPStatusError | None = None
         for attempt in range(retries):
@@ -227,11 +143,8 @@ class Client:
                 )
 
     def close(self):
-        if self._client:
-            self._client.close()
+        self._client.close()
 
     @property
     def base_url(self) -> str:
-        if self.is_offline_mode:
-            return "offline://localhost"
-        return str(self._client.base_url) if self._client else ""
+        return str(self._client.base_url)

{recurvedata_lib-0.1.488 → recurvedata_lib-0.1.490}/recurvedata/connectors/config_schema.py

@@ -1073,84 +1073,6 @@ ALL_CONFIG_SCHEMA_DCT = {
         "group": ["destination"],
         "test_required": True,
     },
-    "recurvedata.connectors.connectors.n8n": {
-        "type": "n8n",
-        "ui_type": "n8n",
-        "category": ["others"],
-        "config_schema": {
-            "type": "object",
-            "properties": {
-                "url": {
-                    "type": "string",
-                    "title": _l("API Address"),
-                    "description": _l("The URL of the n8n API, e.g. https://localhost:5678/api/v1"),
-                },
-                "api_key": {"type": "string", "title": _l("API KEY")},
-                "timeout": {
-                    "type": "number",
-                    "title": _l("Timeout"),
-                    "description": _l("The timeout of the n8n API, e.g. 60"),
-                    "default": 60,
-                },
-                "webhook_credential": {
-                    "type": "object",
-                    "title": _l("Webhook Trigger Node Credential"),
-                    "description": _l("The credential of the n8n webhook trigger node"),
-                    "properties": {
-                        "credential_type": {
-                            "type": "string",
-                            "title": _l("Credential Type"),
-                            "enum": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
-                            "enumNames": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
-                            "default": "None",
-                        },
-                        "basic_auth": {
-                            "ui:hidden": '{{parentFormData.credential_type !== "Basic Auth"}}',
-                            "type": "object",
-                            "title": _l("Basic Auth"),
-                            "description": _l("The basic auth of the n8n webhook trigger node"),
-                            "properties": {
-                                "username": {"type": "string", "title": _l("Username")},
-                                "password": {"type": "string", "title": _l("Password")},
-                            },
-                        },
-                        "header_auth": {
-                            "ui:hidden": '{{parentFormData.credential_type !== "Header Auth"}}',
-                            "type": "object",
-                            "title": _l("Header Auth"),
-                            "description": _l("The header auth of the n8n webhook trigger node"),
-                            "properties": {
-                                "header_name": {"type": "string", "title": _l("Header Name")},
-                                "header_value": {"type": "string", "title": _l("Header Value")},
-                            },
-                        },
-                        "jwt_auth": {
-                            "ui:hidden": '{{parentFormData.credential_type !== "JWT Auth"}}',
-                            "type": "object",
-                            "title": _l("JWT Auth"),
-                            "description": _l("The jwt auth of the n8n webhook trigger node"),
-                            "properties": {
-                                "jwt_token": {"type": "string", "title": _l("JWT Token")},
-                            },
-                        },
-                    },
-                    "order": ["credential_type", "basic_auth", "header_auth", "jwt_auth"],
-                },
-                "proxies": HTTP_PROXY_CONFIG_SCHEMA["proxies"],
-            },
-            "order": ["url", "api_key", "timeout", "webhook_credential", "proxies"],
-            "required": ["url", "api_key"],
-            "secret": [
-                "api_key",
-                "webhook_credential.basic_auth.password",
-                "webhook_credential.header_auth.header_value",
-                "webhook_credential.jwt_auth.jwt_token",
-            ],
-        },
-        "enabled": True,
-        "group": ["destination"],
-        "test_required": True,
-    },
     "recurvedata.connectors.connectors.oss": {
         "type": "oss",
         "ui_type": "Aliyun OSS",

{recurvedata_lib-0.1.488 → recurvedata_lib-0.1.490}/recurvedata/schedulers/airflow_db_process.py

@@ -182,14 +182,19 @@ class AirflowDbService:
         include_downstream: bool,
         failed_only: bool,
     ):
-        drs: list[DagRun] = DagRun.find(
-            dag_id=dag.dag_id,
-            run_id=run_id,
-            execution_start_date=min_execution_date,
-            execution_end_date=max_execution_date,
-        )
+        kwargs = {
+            "dag_id": dag.dag_id,
+            "run_id": run_id,
+            "execution_start_date": min_execution_date,
+            "execution_end_date": max_execution_date,
+        }
+        if min_execution_date or max_execution_date:
+            kwargs.pop("run_id")
+        drs: list[DagRun] = DagRun.find(**kwargs)
         if not drs:
-            logger.info(f"skip rerun, no dag_run found for {dag.dag_id} at {run_id}")
+            logger.info(
+                f"skip rerun, no dag_run found for {dag.dag_id} at {run_id}, or between {min_execution_date} and {max_execution_date}"
+            )
             return
         clear_start_date = min([dr.execution_date for dr in drs])
         clear_end_date = max([dr.execution_date for dr in drs])
@@ -205,14 +210,16 @@ class AirflowDbService:
                 for task_id in clear_task_ids:
                     if task_id in dag.task_dict:
                         task = dag.task_dict[task_id]
-                        upstream_task_ids = [t.task_id for t in task.upstream_list]
+                        # Use Airflow's built-in method to get all upstream tasks
+                        upstream_task_ids = task.get_flat_relative_ids(upstream=True)
                         expanded_task_ids.update(upstream_task_ids)
 
             if include_downstream:
                 for task_id in clear_task_ids:
                     if task_id in dag.task_dict:
                         task = dag.task_dict[task_id]
-                        downstream_task_ids = [t.task_id for t in task.downstream_list]
+                        # Use Airflow's built-in method to get all downstream tasks
+                        downstream_task_ids = task.get_flat_relative_ids(upstream=False)
                         expanded_task_ids.update(downstream_task_ids)
 
         clear_task_ids = list(expanded_task_ids)

recurvedata_lib-0.1.488/recurvedata/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.1.488"

recurvedata_lib-0.1.488/recurvedata/connectors/connectors/n8n.py

@@ -1,141 +0,0 @@
-import base64
-from contextlib import contextmanager
-from urllib.parse import urlparse
-
-import httpx
-
-from recurvedata.connectors._register import register_connector_class
-from recurvedata.connectors.base import RecurveConnectorBase
-from recurvedata.connectors.proxy import HTTP_PROXY_CONFIG_SCHEMA, HttpProxyMixin
-from recurvedata.consts import ConnectorGroup
-from recurvedata.core.translation import _l
-
-CONNECTION_TYPE = "n8n"
-UI_CONNECTION_TYPE = "n8n"
-
-
-@register_connector_class([CONNECTION_TYPE, UI_CONNECTION_TYPE])
-class N8N(HttpProxyMixin, RecurveConnectorBase):
-    connection_type = CONNECTION_TYPE
-    ui_connection_type = UI_CONNECTION_TYPE
-    group = [ConnectorGroup.DESTINATION]
-
-    config_schema = {
-        "type": "object",
-        "properties": {
-            "url": {
-                "type": "string",
-                "title": _l("API Address"),
-                "description": _l("The URL of the n8n API, e.g. https://localhost:5678/api/v1"),
-            },
-            "api_key": {"type": "string", "title": _l("API KEY")},
-            "timeout": {
-                "type": "number",
-                "title": _l("Timeout"),
-                "description": _l("The timeout of the n8n API, e.g. 60"),
-                "default": 60,
-            },
-            "webhook_credential": {
-                "type": "object",
-                "title": _l("Webhook Trigger Node Credential"),
-                "description": _l("The credential of the n8n webhook trigger node"),
-                "properties": {
-                    "credential_type": {
-                        "type": "string",
-                        "title": _l("Credential Type"),
-                        "enum": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
-                        "enumNames": ["Basic Auth", "Header Auth", "JWT Auth", "None"],
-                        "default": "None",
-                    },
-                    "basic_auth": {
-                        "ui:hidden": '{{parentFormData.credential_type !== "Basic Auth"}}',
-                        "type": "object",
-                        "title": _l("Basic Auth"),
-                        "description": _l("The basic auth of the n8n webhook trigger node"),
-                        "properties": {
-                            "username": {"type": "string", "title": _l("Username")},
-                            "password": {"type": "string", "title": _l("Password")},
-                        },
-                    },
-                    "header_auth": {
-                        "ui:hidden": '{{parentFormData.credential_type !== "Header Auth"}}',
-                        "type": "object",
-                        "title": _l("Header Auth"),
-                        "description": _l("The header auth of the n8n webhook trigger node"),
-                        "properties": {
-                            "header_name": {"type": "string", "title": _l("Header Name")},
-                            "header_value": {"type": "string", "title": _l("Header Value")},
-                        },
-                    },
-                    "jwt_auth": {
-                        "ui:hidden": '{{parentFormData.credential_type !== "JWT Auth"}}',
-                        "type": "object",
-                        "title": _l("JWT Auth"),
-                        "description": _l("The jwt auth of the n8n webhook trigger node"),
-                        "properties": {
-                            "jwt_token": {"type": "string", "title": _l("JWT Token")},
-                        },
-                    },
-                },
-                "order": ["credential_type", "basic_auth", "header_auth", "jwt_auth"],
-            },
-            "proxies": HTTP_PROXY_CONFIG_SCHEMA["proxies"],
-        },
-        "order": ["url", "api_key", "timeout", "webhook_credential", "proxies"],
-        "required": ["url", "api_key"],
-        "secret": [
-            "api_key",
-            "webhook_credential.basic_auth.password",
-            "webhook_credential.header_auth.header_value",
-            "webhook_credential.jwt_auth.jwt_token",
-        ],
-    }
-
-    def test_connection(self):
-        pass
-
-    @contextmanager
-    def _n8n_client(self) -> httpx.Client:
-        with self._init_proxy_manager():
-            yield httpx.Client(
-                base_url=f"{self.url}", headers={"X-N8N-API-KEY": f"{self.api_key}"}, timeout=self.timeout
-            )
-
-    def get_workflows(self) -> list[dict]:
-        path = "/workflows"
-        workflows = []
-        cursor = None
-        with self._n8n_client() as client:
-            response = client.get(path)
-            workflows.extend(response.json()["data"])
-            if response.json()["nextCursor"] and response.json()["nextCursor"] != cursor:
-                cursor = response.json()["nextCursor"]
-                while cursor:
-                    response = client.get(path, params={"cursor": cursor})
-                    workflows.extend(response.json()["data"])
-                    cursor = response.json()["nextCursor"]
-            return workflows
-
-    def _trigger_workflow_via_webhook(self, webhook_id: str, payload: dict) -> dict:
-        main_url = f"{urlparse(self.url).scheme}://{urlparse(self.url).netloc}"
-        webhook_url = f"{main_url}/webhook/{webhook_id}"
-        headers = {}
-        credential_type = self.webhook_credential.get("credential_type")
-        basic_auth = self.webhook_credential.get("basic_auth", {})
-        header_auth = self.webhook_credential.get("header_auth", {})
-        jwt_auth = self.webhook_credential.get("jwt_auth", {})
-        with self._init_proxy_manager():
-            if credential_type == "Basic Auth":
-                username = basic_auth.get("username", "")
-                password = basic_auth.get("password", "")
-                headers["Authorization"] = f'Basic {base64.b64encode(f"{username}:{password}".encode()).decode()}'
-            elif credential_type == "Header Auth":
-                header_name = header_auth.get("header_name", "")
-                header_value = header_auth.get("header_value", "")
-                headers[header_name] = header_value
-            elif credential_type == "JWT Auth":
-                jwt_token = jwt_auth.get("jwt_token", "")
-                headers["Authorization"] = f"Bearer {jwt_token}"
-            response = httpx.post(url=webhook_url, headers=headers, timeout=self.timeout, json=payload)
-
-            return response.json()