dataops-sdk 0.3.0__py3-none-any.whl

dataops_sdk/runner/utils/sftp_file_transfer.py

@@ -0,0 +1,243 @@
+"""
+基于 SFTP 的文件传输实现。
+
+继承自 FileTransferBase，使用 paramiko 实现 SFTP 文件传输。
+需要安装 paramiko 依赖：pip install paramiko>=3.0.0
+
+Classes:
+    SFTPFileTransfer: 基于 paramiko 的 SFTP 文件传输类
+
+Example:
+    >>> # 使用密码认证
+    >>> with SFTPFileTransfer(host="sftp.example.com", port=22,
+    ...                        user="sftp_user", password="sftp_password",
+    ...                        remote_path="/uploads") as sftp:
+    ...     sftp.upload("/local/file.txt", "remote_file.txt")
+    ...     files = sftp.list_files("*.txt")
+    ...
+    >>> # 使用 SSH 密钥认证（password=None）
+    >>> with SFTPFileTransfer(host="sftp.example.com", port=22,
+    ...                        user="sftp_user", password=None,
+    ...                        remote_path="/uploads") as sftp:
+    ...     sftp.upload("/local/file.txt", "remote_file.txt")
+"""
+
+import fnmatch
+from io import BytesIO
+from pathlib import Path
+from typing import BinaryIO
+
+import paramiko
+
+from .file_transfer_base import FileTransferBase
+
+
+class SFTPFileTransfer(FileTransferBase):
+    """
+    基于 paramiko 的 SFTP 文件传输类。
+
+    支持：
+    - 上传/下载文件（支持 SSH 密钥认证）
+    - 列出远程文件（支持 glob 模式过滤）
+    - 删除远程文件
+    - 检查文件是否存在
+    - 连接健康检查
+    - 上下文管理器（自动关闭连接）
+
+    继承自 FileTransferBase，实现所有抽象方法。
+
+    Attributes:
+        _ssh: paramiko.SSHClient 实例
+        _sftp: paramiko.SFTPClient 实例
+
+    Example:
+        >>> # 使用密码认证
+        >>> with SFTPFileTransfer(host="sftp.example.com", port=22,
+        ...                        user="sftp_user", password="sftp_password",
+        ...                        remote_path="/uploads") as sftp:
+        ...     sftp.upload("/local/file.txt", "remote_file.txt")
+        ...     files = sftp.list_files("*.txt")
+        ...
+        >>> # 使用 SSH 密钥认证
+        >>> with SFTPFileTransfer(host="sftp.example.com", port=22,
+        ...                        user="sftp_user", password=None,
+        ...                        remote_path="/uploads") as sftp:
+        ...     sftp.upload("/local/file.txt", "remote_file.txt")
+    """
+
+    def post_init(self):
+        """
+        初始化 SFTP 连接。
+
+        连接到 SFTP 服务器并切换到远程基础目录。
+        支持密码认证和 SSH 密钥认证（password=None 时使用默认 SSH 密钥）。
+
+        Returns:
+            None
+        """
+        self.logger.info(f"Connecting to SFTP server: {self._host}:{self._port}")
+
+        self._ssh: paramiko.SSHClient = paramiko.SSHClient()
+        self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        # 连接认证
+        if self._password:
+            self._ssh.connect(
+                hostname=self._host,
+                port=self._port,
+                username=self._user,
+                password=self._password,
+            )
+        else:
+            # 尝试使用默认 SSH 密钥
+            self._ssh.connect(
+                hostname=self._host,
+                port=self._port,
+                username=self._user,
+            )
+
+        self._sftp = self._ssh.open_sftp()
+
+        # 切换到远程基础目录
+        self._sftp.chdir(self._remote_path)
+
+        self.logger.info(f"SFTP connection established to {self._host}")
+
+    def upload(self, fp: BinaryIO | str | bytes, remote_name: str) -> bool:
+        """
+        上传文件到 SFTP 服务器。
+
+        Args:
+            fp: 文件对象，支持任意 IO 类型（如 BytesIO, FileIO 等）。
+            remote_name: 远程文件名。
+
+        Returns:
+            bool: 上传成功返回 True，否则返回 False。
+        """
+        try:
+            if isinstance(fp, str):
+                fp = BytesIO(fp.encode("utf-8"))
+            elif isinstance(fp, bytes):
+                fp = BytesIO(fp)
+            self._sftp.putfo(fp, remote_name)
+            self.logger.info(f"Uploaded: -> {remote_name}")
+            return True
+        except Exception as e:
+            self.logger.error(f"Failed to upload: {e}")
+            return False
+
+    def download(self, remote_name: str, local_path: str) -> bool:
+        """
+        从 SFTP 服务器下载文件。
+
+        Args:
+            remote_name: 远程文件名。
+            local_path: 本地保存路径。
+
+        Returns:
+            bool: 下载成功返回 True，否则返回 False。
+        """
+        try:
+            # 确保本地目录存在
+            Path(local_path).parent.mkdir(parents=True, exist_ok=True)
+
+            self._sftp.get(remote_name, local_path)
+            self.logger.info(f"Downloaded: {remote_name} -> {local_path}")
+            return True
+        except Exception as e:
+            self.logger.error(f"Failed to download {remote_name}: {e}")
+            return False
+
+    def list_files(self, pattern: str | None = None) -> list[str]:
+        """
+        列出 SFTP 远程目录下的文件。
+
+        Args:
+            pattern: 可选的 glob 模式，用于过滤文件名。
+
+        Returns:
+            list[str]: 文件名列表。
+        """
+        try:
+            files = self._sftp.listdir()
+            if pattern:
+                files = [f for f in files if fnmatch.fnmatch(f, pattern)]
+            return files
+        except Exception as e:
+            self.logger.error(f"Failed to list files: {e}")
+            return []
+
+    def delete(self, remote_name: str) -> bool:
+        """
+        删除 SFTP 服务器上的文件。
+
+        Args:
+            remote_name: 远程文件名。
+
+        Returns:
+            bool: 删除成功返回 True，否则返回 False。
+        """
+        try:
+            self._sftp.remove(remote_name)
+            self.logger.info(f"Deleted: {remote_name}")
+            return True
+        except Exception as e:
+            self.logger.error(f"Failed to delete {remote_name}: {e}")
+            return False
+
+    def exists(self, remote_name: str) -> bool:
+        """
+        检查 SFTP 服务器上文件是否存在。
+
+        Args:
+            remote_name: 远程文件名。
+
+        Returns:
+            bool: 文件存在返回 True，否则返回 False。
+        """
+        try:
+            self._sftp.stat(remote_name)
+            return True
+        except FileNotFoundError:
+            return False
+        except Exception as e:
+            self.logger.error(f"Failed to check existence of {remote_name}: {e}")
+            return False
+
+    def ping(self) -> bool:
+        """
+        检查 SFTP 连接是否存活。
+
+        Returns:
+            bool: 连接正常返回 True，否则返回 False。
+        """
+        try:
+            # 执行一个简单的 SFTP 操作来检查连接
+            channel = self._sftp.get_channel()
+            return channel is not None and not channel.closed
+        except Exception as e:
+            self.logger.warning(f"SFTP ping failed: {e}")
+            return False
+
+    def close(self):
+        """
+        关闭 SFTP 连接。
+
+        关闭 SFTP 和 SSH 连接，释放资源。
+        支持多次调用。
+        """
+        if hasattr(self, "_sftp") and self._sftp is not None:
+            try:
+                self._sftp.close()
+                self.logger.info("SFTP channel closed")
+            except Exception as e:
+                self.logger.error(f"Failed to close SFTP channel: {e}")
+            self._sftp = None
+
+        if hasattr(self, "_ssh") and self._ssh is not None:
+            try:
+                self._ssh.close()
+                self.logger.info("SSH connection closed")
+            except Exception as e:
+                self.logger.error(f"Failed to close SSH connection: {e}")
+            self._ssh = None

dataops_sdk/schemas/__init__.py

@@ -0,0 +1,32 @@
+from .alarm import AlarmRuleItem
+from .api import ApiMeta, HttpInterfaceData
+from .data_table import DataTableData, MetaDataTable
+from .datasource import DataDataSource
+from .dict_ import DataDict, DictItem
+from .enums import InnerMetaType, MetaType
+from .group import GroupNode
+from .hazard import HazardRecordDTO, HazardTaskDTO
+from .meta import (
+    DEFAULT_META_DATA_TYPE,
+    META_DATA_TYPE,
+    Meta,
+    MetaBase,
+    MetaCreate,
+    MetaRef,
+    MetaRelease,
+    MetaUpdate,
+)
+from .query import PageParam, SqlCondition, SqlQueryParam, SqlSort
+from .report import ReportMeta, ReportMetaData
+from .skill import DataSkill, MetaSkill, SkillCallResult
+from .table import (
+    ColumnItem,
+    DataBaseColumnDictExternal,
+    DataBaseColumnReference,
+    DataBaseColumnReferenceValue,
+    DataBaseColumnTableExternal,
+    DataDataBase,
+)
+from .user import Application, UserInfo
+from .widget import WidgetMeta, WidgetMetaData
+from .workflow import DataWorkflow, MetaWorkflow

dataops_sdk/schemas/alarm.py

@@ -0,0 +1,38 @@
+from typing import Annotated
+
+from pydantic import BaseModel, Field
+
+from dataops_sdk.schemas.meta import MetaRelease
+
+
+class AlarmExternal(BaseModel):
+    IS_ENABLED: Annotated[str, Field(description="是否启用")]
+    N_IS_ENABLED: Annotated[str, Field(description="是否启用名称")]
+
+
+class AlarmRuleItem(MetaRelease, AlarmExternal):
+    INDUSTRY: Annotated[str, Field(description="行业")]
+    DESCRIPTION: Annotated[str, Field(description="描述")] = ""
+
+
+class AlarmSpecItem(MetaRelease, AlarmExternal):
+    INDUSTRY: Annotated[str, Field(description="行业")]
+    N_INDUSTRY: Annotated[str, Field(description="行业名称")]
+    BUSINESS_DOMAIN: Annotated[str, Field(description="业务域")]
+    N_BUSINESS_DOMAIN: Annotated[str, Field(description="业务域名称")]
+    KAFKA_TOPICS: Annotated[str, Field(description="Kafka主题列表")]
+
+
+class AlarmActionItem(MetaRelease, AlarmExternal):
+    RULE_TYPE: Annotated[str, Field(description="规则类型")]
+    N_RULE_TYPE: Annotated[str, Field(description="规则类型名称")]
+
+    SCRIPT_CONTENT: Annotated[str, Field(description="脚本内容")]
+
+
+class AlarmRuleMatchItem(MetaRelease, AlarmExternal):
+    AREA: Annotated[str, Field(description="区域")]
+    SPEC_TYPE: Annotated[str, Field(description="规则类型")]
+    N_SPEC_TYPE: Annotated[str, Field(description="规则类型名称")]
+    RULE_TYPE: Annotated[str, Field(description="规则类型")]
+    N_RULE_TYPE: Annotated[str, Field(description="规则类型名称")]

dataops_sdk/schemas/api.py

@@ -0,0 +1,37 @@
+from collections.abc import Mapping
+from typing import Annotated
+
+from pydantic import BaseModel, Field, JsonValue
+
+from .meta import Meta
+
+
+class HttpInterfaceData(BaseModel):
+    """API 接口元数据.
+
+    对应前端 types/restful.ts 中的 HttpInterface 类型。
+    """
+
+    endpoint: Annotated[str, Field(description="接口路径")]
+    method: Annotated[str, Field(description="HTTP 方法: GET/POST/PUT/DELETE/PATCH/HEAD/OPTIONS")]
+    status: Annotated[str, Field(description="接口状态: dev/test/prod/deprecated")]
+    scope: Annotated[str, Field(description="接口范围: public/internal/external")]
+    description: Annotated[str, Field(description="接口说明，支持 Markdown")] = ""
+    charger: Annotated[str | None, Field(description="负责人")] = None
+    env_code: Annotated[str | None, Field(description="关联环境编码")] = None
+    requestParam: Annotated[
+        Mapping[str, JsonValue],
+        Field(description="请求参数（query/path/body/headers/cookies/auth）"),
+    ]
+    responseParams: Annotated[
+        list[Mapping[str, JsonValue]], Field(description="响应参数定义", default_factory=list)
+    ]
+    response: Annotated[Mapping[str, JsonValue] | None, Field(description="调试响应快照")] = None
+    debugExamples: Annotated[
+        list[Mapping[str, JsonValue]] | None,
+        Field(description="调试测试用例", default_factory=list),
+    ]
+
+
+class ApiMeta(Meta[HttpInterfaceData]):
+    """API Meta"""