dataops-sdk 0.3.0__py3-none-any.whl

dataops_sdk/__init__.py

@@ -0,0 +1,84 @@
+from . import runner, schemas, utils
+from .services import (  # noqa: F401
+    release_list_create,
+    release_list_get_by_code,
+    release_list_get_by_name,
+    release_list_list,
+    release_list_page,
+    release_list_total,
+    release_list_update,
+    release_ref_list,
+    release_ref_list_by_ref,
+    service_clear_hazard_history_records,
+    service_count_alarm_action,
+    service_count_alarm_rule_matches,
+    service_count_alarm_rules,
+    service_count_alarm_specs,
+    service_count_table_by_code,
+    service_count_table_by_name,
+    service_count_widget_by_code,
+    service_create_data_table,
+    service_create_data_table_record,
+    service_delete_data_table_record_by_id,
+    service_delete_data_table_record_by_id_list,
+    service_exec_sql_by_datasource,
+    service_find_alarm_table_data,
+    service_find_meta_by_codes,
+    service_get_alarm_rule_enums,
+    service_get_alarm_table_columns,
+    service_get_api_by_code,
+    service_get_api_by_name,
+    service_get_current_application,
+    service_get_data_table_by_code,
+    service_get_data_table_by_name,
+    service_get_data_table_data_by_code,
+    service_get_data_table_data_by_name,
+    service_get_data_table_record_by_id,
+    service_get_data_table_reference_list,
+    service_get_data_table_total,
+    service_get_dataset_data_by_code,
+    service_get_dataset_detail_by_code,
+    service_get_datasource_by_name,
+    service_get_datasource_config_by_application_id,
+    service_get_dict_detail_by_code,
+    service_get_dict_detail_by_name,
+    service_get_hazard_task_log_records,
+    service_get_hazard_task_logs,
+    service_get_main_datasource_config_by_application,
+    service_get_metrics,
+    service_get_rel_table_columns,
+    service_get_report,
+    service_get_table_data_by_code,
+    service_get_table_data_by_name,
+    service_get_table_detail_by_code,
+    service_get_table_detail_by_name,
+    service_get_widget_data_by_code,
+    service_get_widget_detail_by_code,
+    service_get_workflow_by_code,
+    service_get_workflow_by_name,
+    service_list_alarm_rule_matches,
+    service_list_alarm_rules,
+    service_list_alarm_specs,
+    service_list_applications,
+    service_list_datasets,
+    service_list_dicts,
+    service_list_hazard_history_records,
+    service_list_hazard_records,
+    service_list_metrics,
+    service_list_reports,
+    service_list_tables,
+    service_list_widgets,
+    service_list_workflows,
+    service_login,
+    service_logout,
+    service_ref_backward,
+    service_ref_forward,
+    service_set_application,
+    service_start_workflow,
+    service_start_workflow_async,
+    service_trigger_hazard_task,
+    service_update_data_table_record_by_id,
+    service_update_data_table_record_by_id_list,
+)
+
+__version__ = "0.3.0"

dataops_sdk/constants.py

@@ -0,0 +1,6 @@
+from typing import Final
+
+USER_INFO_FILE: Final[str] = "user_info.json"
+APPLICATION_FILE: Final[str] = "application.json"
+
+DEFAULT_SECRET_KEY: Final[str] = "e0XxEhkWSJcA6Aps"

dataops_sdk/exceptions.py

@@ -0,0 +1,31 @@
+class DataOpsError(Exception):
+    """DataOps API 错误基类。"""
+
+    def __init__(self, message: str, status_code: int | None = None):
+        self.message = message
+        self.status_code = status_code
+        super().__init__(message)
+
+
+class NotFoundError(DataOpsError):
+    """资源未找到。"""
+
+    pass
+
+
+class AuthenticationError(DataOpsError):
+    """认证失败异常。"""
+
+    pass
+
+
+class FailedToCreateError(DataOpsError):
+    """创建失败异常。"""
+
+    pass
+
+
+class FailedToUpdateError(DataOpsError):
+    """更新失败异常。"""
+
+    pass

dataops_sdk/runner/__init__.py

@@ -0,0 +1,46 @@
+"""Runner 包 - 独立的脚本执行工具集。
+
+提供脚本执行所需的数据库连接、DuckDB 管理、Neo4j 连接等工具。
+可作为独立模块被导入使用。
+
+Modules:
+    schemas: Nacos 配置数据模型
+    utils: 通用工具（数据库连接、加密、日志等）
+
+Example:
+    >>> from runner.utils import DatabaseConn, DuckDBManager
+"""
+
+from .datasource_factory import get_datasource_instent
+from .schemas.nacos import NacosSettings
+from .utils import (
+    DatabaseConn,
+    FTPFileTransfer,
+    KafkaQueue,
+    Neo4jConn,
+    SFTPFileTransfer,
+    decrypt_body,
+    encrypt_body,
+    generate_uuid,
+    get_service_instance,
+    getLogger,
+    parse_args,
+    print_json,
+)
+
+__all__ = [
+    "DatabaseConn",
+    "FTPFileTransfer",
+    "KafkaQueue",
+    "SFTPFileTransfer",
+    "Neo4jConn",
+    "decrypt_body",
+    "encrypt_body",
+    "getLogger",
+    "print_json",
+    "generate_uuid",
+    "parse_args",
+    "NacosSettings",
+    "get_service_instance",
+    "get_datasource_instent",
+]

dataops_sdk/runner/datasource_factory.py

@@ -0,0 +1,83 @@
+"""
+数据源实例工厂模块。
+
+根据数据源元数据动态创建对应的连接实例（数据库或消息队列）。
+"""
+
+from dataops_sdk.services.datasource import service_get_datasource_by_name
+
+
+def get_datasource_instent(datasource: str, log_sql: bool | None = None):
+    """
+    根据数据源名称获取对应的连接实例。
+
+    通过数据源名称从平台查询元数据，根据数据源的 category（数据库/消息队列）
+    和 type（具体类型）动态创建并返回相应的连接实例。
+
+    Args:
+        datasource: 数据源名称，用于从平台查询数据源元数据。
+        log_sql: 是否记录 SQL 执行日志，传递给数据库连接实例。
+
+    Returns:
+        DatabaseConn: 通用数据库连接实例（MySQL、PostgreSQL 等）。
+        Neo4jConn: Neo4j 图数据库连接实例。
+        KafkaQueue: Kafka 消息队列连接实例。
+
+    Raises:
+        NotImplementedError: 当数据源类型未被支持时抛出。
+
+    Note:
+        当前支持的 category:
+        - database: 关系型/图数据库（Neo4j、MySQL、PostgreSQL 等）
+        - queue: 消息队列（Kafka）
+    """
+    # 从平台查询数据源元数据
+    meta = service_get_datasource_by_name(datasource)
+
+    # 根据数据源类别分发处理
+    if meta.data.category == "queue":
+        # 消息队列类型处理
+        if meta.data.type == "kafka":
+            # Kafka 连接需要 bootstrap_servers
+            assert meta.data.bootstrap_servers, "bootstrap_servers is required, but not provided"
+            from dataops_sdk.runner import KafkaQueue
+
+            return KafkaQueue(meta.data.bootstrap_servers)
+        else:
+            raise NotImplementedError(f"Queue type {meta.data.type} is not supported")
+
+    elif meta.data.category == "database":
+        # 数据库类型处理，验证必要的连接参数
+        assert meta.data.ip, "host is required, but not provided"
+        assert meta.data.port, "port is required, but not provided"
+        assert meta.data.username, "user is required, but not provided"
+        assert meta.data.password, "password is required, but not provided"
+        assert meta.data.database, "database is required, but not provided"
+
+        if meta.data.type == "neo4j":
+            # Neo4j 图数据库使用专用连接类
+            from dataops_sdk.runner import Neo4jConn
+
+            return Neo4jConn(
+                host=meta.data.ip,
+                port=meta.data.port,
+                user=meta.data.username,
+                password=meta.data.password,
+                db=meta.data.database,
+                log_sql=log_sql,
+                name=meta.title,
+            )
+        else:
+            # 通用数据库连接，支持 MySQL、PostgreSQL 等
+            from dataops_sdk.runner import DatabaseConn
+
+            return DatabaseConn(
+                host=meta.data.ip,
+                port=meta.data.port,
+                user=meta.data.username,
+                password=meta.data.password,
+                db=meta.data.database,
+                db_type=meta.data.type,
+                log_sql=log_sql,
+                name=meta.title,
+            )

dataops_sdk/runner/schemas/nacos/__init__.py

@@ -0,0 +1,5 @@
+from .config import NacosSettings
+from .instance import Instance
+from .service import Service
+
+__all__ = ["Service", "Instance", "NacosSettings"]

dataops_sdk/runner/schemas/nacos/config.py

@@ -0,0 +1,40 @@
+"""Nacos 配置模块。
+
+定义 Nacos 相关的配置数据模型，与主应用中的 Settings 类似，
+但用于 runner 子包独立运行时。
+
+Classes:
+    NacosSettings: Nacos 配置类
+
+Example:
+    >>> from runner.schemas.nacos import NacosSettings
+    >>> config = NacosSettings()
+"""
+
+from typing import Annotated
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class NacosSettings(BaseSettings):
+    """Nacos 配置类。
+
+    从 .env 文件和环境变量加载配置。配置项包括后端监听、
+    子系统连接和 Nacos 服务发现参数。
+
+    Attributes:
+        nacos_addrs: NACOS服务地址
+        nacos_user: NACOS用户名
+        nacos_password: NACOS密码
+        nacos_discovery_namespace: NACOS服务发现命名空间
+        nacos_group_name: NACOS服务发现组名
+    """
+
+    nacos_addrs: Annotated[str, Field(description="NACOS服务地址")]
+    nacos_user: Annotated[str, Field(description="NACOS用户名")]
+    nacos_password: Annotated[str, Field(description="NACOS密码")]
+    nacos_discovery_namespace: Annotated[str, Field(description="NACOS服务发现命名空间")]
+    nacos_group_name: Annotated[str, Field(description="NACOS服务发现组名")] = "DEFAULT_GROUP"
+
+    model_config = SettingsConfigDict(extra="ignore")

dataops_sdk/runner/schemas/nacos/constants.py

@@ -0,0 +1,235 @@
+class Constants:
+    NAMING_MODULE = "naming"
+
+    CONFIG_MODULE = "config"
+
+    AI_MODULE = "ai"
+
+    LABEL_SOURCE = "source"
+
+    LABEL_SOURCE_SDK = "sdk"
+
+    LABEL_SOURCE_CLUSTER = "cluster"
+
+    LABEL_MODULE = "module"
+
+    CLIENT_VERSION = "Nacos-Python-Client:v3.2.0"
+
+    DATA_IN_BODY_VERSION = 204
+
+    DEFAULT_GROUP = "DEFAULT_GROUP"
+
+    WEB_CONTEXT = "/nacos"
+
+    APPNAME = "AppName"
+
+    UNKNOWN_APP = "UnknownApp"
+
+    DEFAULT_DOMAINNAME = "commonconfig.config-host.taobao.com"
+
+    DAILY_DOMAINNAME = "commonconfig.taobao.net"
+
+    NULL = ""
+
+    DATAID = "dataId"
+
+    GROUP = "group"
+
+    DEFAULT_HEARTBEAT_INTERVAL = 5
+
+    LAST_MODIFIED = "Last-Modified"
+
+    ACCEPT_ENCODING = "Accept-Encoding"
+
+    CONTENT_ENCODING = "Content-Encoding"
+
+    PROBE_MODIFY_REQUEST = "Listening-Configs"
+
+    PROBE_MODIFY_RESPONSE = "Probe-Modify-Response"
+
+    PROBE_MODIFY_RESPONSE_NEW = "Probe-Modify-Response-New"
+
+    USE_ZIP = "true"
+
+    CONTENT_MD5 = "Content-MD5"
+
+    CONFIG_VERSION = "Config-Version"
+
+    CONFIG_TYPE = "Config-Type"
+
+    ENCRYPTED_DATA_KEY = "Encrypted-Data-Key"
+
+    IF_MODIFIED_SINCE = "If-Modified-Since"
+
+    SPACING_INTERVAL = "client-spacing-interval"
+
+    BASE_PATH = "/v1/cs"
+
+    SERVICE_BASE_PATH = "/v1/ns"
+
+    CONFIG_CONTROLLER_PATH = BASE_PATH + "/configs"
+
+    TOKEN = "token"
+
+    ACCESS_TOKEN = "accessToken"
+
+    TOKEN_TTL = "tokenTtl"
+
+    GLOBAL_ADMIN = "globalAdmin"
+
+    USERNAME = "username"
+
+    TOKEN_REFRESH_WINDOW = "tokenRefreshWindow"
+
+    # second.
+    ASYNC_UPDATE_ADDRESS_INTERVAL = 300
+
+    # second.
+    POLLING_INTERVAL_TIME = 15
+
+    # millisecond.
+    ONCE_TIMEOUT = 2000
+
+    # millisecond.
+    SO_TIMEOUT = 60000
+
+    # millisecond.
+    CONFIG_LONG_POLL_TIMEOUT = 30000
+
+    # millisecond.
+    MIN_CONFIG_LONG_POLL_TIMEOUT = 10000
+
+    # millisecond.
+    CONFIG_RETRY_TIME = 2000
+
+    # Maximum number of retries.
+    MAX_RETRY = 3
+
+    # millisecond.
+    RECV_WAIT_TIMEOUT = ONCE_TIMEOUT * 5
+
+    ENCODE = "UTF-8"
+
+    MAP_FILE = "map-file.js"
+
+    FLOW_CONTROL_THRESHOLD = 20
+
+    FLOW_CONTROL_SLOT = 10
+
+    FLOW_CONTROL_INTERVAL = 1000
+
+    DEFAULT_PROTECT_THRESHOLD = 0.0
+
+    LINE_SEPARATOR = chr(1)
+
+    WORD_SEPARATOR = chr(2)
+
+    LONGPOLLING_LINE_SEPARATOR = "\r\n"
+
+    CLIENT_APPNAME_HEADER = "Client-AppName"
+
+    APP_NAME_HEADER = "AppName"
+
+    CLIENT_REQUEST_TS_HEADER = "Client-RequestTS"
+
+    CLIENT_REQUEST_TOKEN_HEADER = "Client-RequestToken"
+
+    ATOMIC_MAX_SIZE = 1000
+
+    NAMING_INSTANCE_ID_SPLITTER = "#"
+
+    NAMING_INSTANCE_ID_SEG_COUNT = 4
+
+    NAMING_HTTP_HEADER_SPLITTER = "\\|"
+
+    DEFAULT_CLUSTER_NAME = "DEFAULT"
+
+    DEFAULT_HEART_BEAT_TIMEOUT = 15000
+
+    DEFAULT_IP_DELETE_TIMEOUT = 30000
+
+    DEFAULT_HEART_BEAT_INTERVAL = 5000
+
+    DEFAULT_NAMESPACE_ID = ""
+
+    NAMING_DEFAULT_NAMESPACE_ID = "public"
+
+    DEFAULT_USE_CLOUD_NAMESPACE_PARSING = True
+
+    WRITE_REDIRECT_CODE = 307
+
+    RESPONSE_SUCCESS_CODE = 200
+
+    REQUEST_DOMAIN_RETRY_TIME = 3
+
+    SERVICE_INFO_SPLITER = "@@"
+
+    CONFIG_INFO_SPLITER = "@@"
+
+    SERVICE_INFO_SPLIT_COUNT = 2
+
+    NULL_STRING = "null"
+
+    NUMBER_PATTERN = "^\\d+$"
+
+    ANY_PATTERN = ".*"
+
+    DEFAULT_INSTANCE_ID_GENERATOR = "simple"
+
+    SNOWFLAKE_INSTANCE_ID_GENERATOR = "snowflake"
+
+    HTTP_PREFIX = "http"
+
+    ALL_PATTERN = "*"
+
+    COLON = ":"
+
+    LINE_BREAK = "\n"
+
+    POUND = "#"
+
+    VIPSERVER_TAG = "Vipserver-Tag"
+
+    AMORY_TAG = "Amory-Tag"
+
+    LOCATION_TAG = "Location-Tag"
+
+    CHARSET_KEY = "charset"
+
+    EX_CONFIG_INFO = "exConfigInfo"
+
+    GROUP_NAME_KEY = "groupName"
+
+    SERVICE_NAME_KEY = "serviceName"
+
+    CIPHER_PRE_FIX = "cipher-"
+
+    DEFAULT_PORT = 8848
+
+    GRPC_MAX_RECEIVE_MESSAGE_LENGTH = 100 * 1024 * 1024
+
+    GRPC_KEEPALIVE_TIME_MILLS = 60 * 1000
+
+    GRPC_INITIAL_WINDOW_SIZE = 10 * 1024 * 1024
+
+    GRPC_INITIAL_CONN_WINDOW_SIZE = 10 * 1024 * 1024
+
+    KEEP_ALIVE_TIME_MILLS = 5000
+
+    INTERNAL_TIME_MILLS = 3000
+
+    DEFAULT_GRPC_TIMEOUT_MILLS = 3000
+
+    GRPC_CAPABILITY_NEGOTIATION_TIMEOUT = 5000
+
+    GRPC_PORT_OFFSET = 1000  # gRPC port = HTTP port + offset
+
+    DEFAULT_TIMEOUT_MILLS = 10000
+
+    PER_TASK_CONFIG_SIZE = 3000
+
+    MSE_KMS_V1_DEFAULT_KEY_ID = "alias/acs/mse"
+
+    KMS_AES_128_ALGORITHM_NAME = "AES_128"
+
+    KMS_AES_256_ALGORITHM_NAME = "AES_256"

dataops_sdk/runner/schemas/nacos/instance.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import re
+from typing import Annotated
+
+from pydantic import BaseModel, Field
+
+from .constants import Constants
+from .nacos_exception import INVALID_PARAM, NacosException
+from .preserved_metadata_key import PreservedMetadataKeys
+
+
+class Instance(BaseModel):
+    instanceId: Annotated[str, Field(description="实例ID")] = ""
+    ip: Annotated[str, Field(description="实例IP地址")]
+    port: Annotated[int, Field(description="实例端口号")]
+    weight: Annotated[float, Field(description="实例权重")] = 1.0
+    healthy: Annotated[bool, Field(description="实例健康状态")] = True
+    enabled: Annotated[bool, Field(description="实例是否启用")] = True
+    ephemeral: Annotated[bool, Field(description="实例是否临时实例")] = True
+    clusterName: Annotated[str, Field(description="实例所在集群名称")] = ""
+    serviceName: Annotated[str, Field(description="实例所在服务名称")] = ""
+    metadata: Annotated[dict, Field(description="实例元数据")] = {}
+
+    def __str__(self):
+        return f"Instance({', '.join(f'{key}={value!r}' for key, value in self.__dict__.items())})"
+
+    def to_inet_addr(self):
+        return self.ip + ":" + str(self.port)
+
+    def is_ephemeral(self) -> bool:
+        return self.ephemeral
+
+    def get_weight(self):
+        return self.weight
+
+    def add_metadata(self, key: str, value: str) -> None:
+        if self.metadata is None:
+            self.metadata = {}
+        self.metadata[key] = value
+
+    def get_instance_heart_beat_interval(self):
+        return self.__get_metadata_by_key_with_int_default(
+            PreservedMetadataKeys.HEART_BEAT_INTERVAL, Constants.DEFAULT_HEART_BEAT_INTERVAL
+        )
+
+    def get_instance_heart_beat_timeout(self):
+        return self.__get_metadata_by_key_with_int_default(
+            PreservedMetadataKeys.HEART_BEAT_INTERVAL, Constants.DEFAULT_HEART_BEAT_TIMEOUT
+        )
+
+    def get_ip_delete_timeout(self):
+        return self.__get_metadata_by_key_with_int_default(
+            PreservedMetadataKeys.IP_DELETE_TIMEOUT, Constants.DEFAULT_IP_DELETE_TIMEOUT
+        )
+
+    def get_instance_id_generator(self):
+        return self.__get_metadata_by_key_with_str_default(
+            PreservedMetadataKeys.INSTANCE_ID_GENERATOR, Constants.DEFAULT_INSTANCE_ID_GENERATOR
+        )
+
+    def check_instance_is_legal(self):
+        if (
+            self.get_instance_heart_beat_timeout() < self.get_instance_heart_beat_interval()
+            or self.get_ip_delete_timeout() < self.get_instance_heart_beat_interval()
+        ):
+            raise NacosException(
+                INVALID_PARAM,
+                "Instance 'heart beat interval' must less than 'heart beat timeout' and 'ip delete timeout'.",
+            )
+        self.fill_default_value()
+
+        if not self.ip or len(self.ip) == 0:
+            raise NacosException(INVALID_PARAM, "Instance 'ip' can not be empty.")
+
+        if self.port < 0 or self.port > 65535:
+            raise NacosException(INVALID_PARAM, "Instance 'port' must be between 0 and 65535.")
+
+    def fill_default_value(self):
+        if self.clusterName is None or len(self.clusterName) == 0:
+            self.clusterName = Constants.DEFAULT_CLUSTER_NAME
+
+    def contains_metadata(self, key: str) -> bool:
+        if not self.metadata:
+            return False
+        return key in self.metadata.keys()
+
+    def __get_metadata_by_key_with_int_default(self, key: str, default_value: int) -> int:
+        if not self.metadata or key not in self.metadata:
+            return default_value
+        value = self.metadata[key]
+
+        pattern = re.compile(Constants.NUMBER_PATTERN)
+
+        if value.strip() and re.match(pattern, value):
+            return int(value)
+
+        return default_value
+
+    def __get_metadata_by_key_with_str_default(self, key: str, default_value: str) -> str:
+        if not self.metadata:
+            return default_value
+        return self.metadata[key]

dataops_sdk/runner/schemas/nacos/nacos_exception.py

@@ -0,0 +1,44 @@
+CLIENT_INVALID_PARAM = -400
+
+CLIENT_DISCONNECT = -401
+
+CLIENT_OVER_THRESHOLD = -503
+
+INVALID_PARAM = 400
+
+NO_RIGHT = 403
+
+NOT_FOUND = 404
+
+CONFLICT = 409
+
+SERVER_ERROR = 500
+
+SERVER_NOT_IMPLEMENTED = 501
+
+BAD_GATEWAY = 502
+
+OVER_THRESHOLD = 503
+
+INVALID_SERVER_STATUS = 300
+
+UN_REGISTER = 301
+
+NO_HANDLER = 302
+
+NOT_MODIFIED = 304
+
+INVALID_INTERFACE_ERROR = -403
+
+RESOURCE_NOT_FOUND = -404
+
+HTTP_CLIENT_ERROR_CODE = -500
+
+
+class NacosException(Exception):
+    """Custom exception class with an error code attribute."""
+
+    def __init__(self, error_code, message="An error occurred"):
+        self.error_code = error_code
+        self.message = message
+        super().__init__(f"Error [{error_code}]: {message}")