dataops-sdk 0.3.0__tar.gz

dataops_sdk-0.3.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: dataops-sdk
+Version: 0.3.0
+Summary: Python SDK for the DataOps platform.
+Requires-Python: >=3.11
+Requires-Dist: pydantic<3,>=2
+Requires-Dist: pydantic-core
+Requires-Dist: pydantic-settings>=2.14.2
+Requires-Dist: httpx2>=2.5.0
+Requires-Dist: jinja2>=3.1.6
+Requires-Dist: markdown-it-py<4,>=3
+Requires-Dist: cryptography>=49.0.0
+Provides-Extra: runner
+Requires-Dist: paramiko>4; extra == "runner"
+Requires-Dist: pandas; extra == "runner"
+Requires-Dist: sqlalchemy; extra == "runner"
+Requires-Dist: pymysql; extra == "runner"
+Requires-Dist: doris-python>=1.0.3; extra == "runner"
+Requires-Dist: oracledb~=4.0.1; extra == "runner"
+Requires-Dist: neo4j; extra == "runner"
+Requires-Dist: kafka-python>3; extra == "runner"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Requires-Dist: ruff; extra == "dev"

dataops_sdk-0.3.0/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dataops-sdk"
+dynamic = ["version"]
+description = "Python SDK for the DataOps platform."
+requires-python = ">=3.11"
+dependencies = [
+    "pydantic>=2,<3",
+    "pydantic-core",
+    "pydantic-settings>=2.14.2",
+    "httpx2>=2.5.0",
+    "jinja2>=3.1.6",
+    "markdown-it-py>=3,<4",
+    "cryptography>=49.0.0",
+]
+
+[project.optional-dependencies]
+runner = [
+    # 文件传输
+    "paramiko>4",
+    # 数据分析
+    "pandas",
+    # 关系数据库
+    "sqlalchemy",
+    "pymysql",
+    "doris-python>=1.0.3",
+    "oracledb~=4.0.1",
+    # 图数据库
+    "neo4j",
+    # 队列驱动
+    "kafka-python>3",
+]
+dev = ["pytest", "pytest-mock", "ruff"]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+include-package-data = true
+
+[tool.setuptools.dynamic]
+version = {attr = "dataops_sdk.__version__"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+dataops_sdk = ["py.typed"]

dataops_sdk-0.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

dataops_sdk-0.3.0/src/dataops_sdk/__init__.py

@@ -0,0 +1,84 @@
+from . import runner, schemas, utils
+from .services import (  # noqa: F401
+    release_list_create,
+    release_list_get_by_code,
+    release_list_get_by_name,
+    release_list_list,
+    release_list_page,
+    release_list_total,
+    release_list_update,
+    release_ref_list,
+    release_ref_list_by_ref,
+    service_clear_hazard_history_records,
+    service_count_alarm_action,
+    service_count_alarm_rule_matches,
+    service_count_alarm_rules,
+    service_count_alarm_specs,
+    service_count_table_by_code,
+    service_count_table_by_name,
+    service_count_widget_by_code,
+    service_create_data_table,
+    service_create_data_table_record,
+    service_delete_data_table_record_by_id,
+    service_delete_data_table_record_by_id_list,
+    service_exec_sql_by_datasource,
+    service_find_alarm_table_data,
+    service_find_meta_by_codes,
+    service_get_alarm_rule_enums,
+    service_get_alarm_table_columns,
+    service_get_api_by_code,
+    service_get_api_by_name,
+    service_get_current_application,
+    service_get_data_table_by_code,
+    service_get_data_table_by_name,
+    service_get_data_table_data_by_code,
+    service_get_data_table_data_by_name,
+    service_get_data_table_record_by_id,
+    service_get_data_table_reference_list,
+    service_get_data_table_total,
+    service_get_dataset_data_by_code,
+    service_get_dataset_detail_by_code,
+    service_get_datasource_by_name,
+    service_get_datasource_config_by_application_id,
+    service_get_dict_detail_by_code,
+    service_get_dict_detail_by_name,
+    service_get_hazard_task_log_records,
+    service_get_hazard_task_logs,
+    service_get_main_datasource_config_by_application,
+    service_get_metrics,
+    service_get_rel_table_columns,
+    service_get_report,
+    service_get_table_data_by_code,
+    service_get_table_data_by_name,
+    service_get_table_detail_by_code,
+    service_get_table_detail_by_name,
+    service_get_widget_data_by_code,
+    service_get_widget_detail_by_code,
+    service_get_workflow_by_code,
+    service_get_workflow_by_name,
+    service_list_alarm_rule_matches,
+    service_list_alarm_rules,
+    service_list_alarm_specs,
+    service_list_applications,
+    service_list_datasets,
+    service_list_dicts,
+    service_list_hazard_history_records,
+    service_list_hazard_records,
+    service_list_metrics,
+    service_list_reports,
+    service_list_tables,
+    service_list_widgets,
+    service_list_workflows,
+    service_login,
+    service_logout,
+    service_ref_backward,
+    service_ref_forward,
+    service_set_application,
+    service_start_workflow,
+    service_start_workflow_async,
+    service_trigger_hazard_task,
+    service_update_data_table_record_by_id,
+    service_update_data_table_record_by_id_list,
+)
+
+__version__ = "0.3.0"

dataops_sdk-0.3.0/src/dataops_sdk/constants.py

@@ -0,0 +1,6 @@
+from typing import Final
+
+USER_INFO_FILE: Final[str] = "user_info.json"
+APPLICATION_FILE: Final[str] = "application.json"
+
+DEFAULT_SECRET_KEY: Final[str] = "e0XxEhkWSJcA6Aps"

dataops_sdk-0.3.0/src/dataops_sdk/exceptions.py

@@ -0,0 +1,31 @@
+class DataOpsError(Exception):
+    """DataOps API 错误基类。"""
+
+    def __init__(self, message: str, status_code: int | None = None):
+        self.message = message
+        self.status_code = status_code
+        super().__init__(message)
+
+
+class NotFoundError(DataOpsError):
+    """资源未找到。"""
+
+    pass
+
+
+class AuthenticationError(DataOpsError):
+    """认证失败异常。"""
+
+    pass
+
+
+class FailedToCreateError(DataOpsError):
+    """创建失败异常。"""
+
+    pass
+
+
+class FailedToUpdateError(DataOpsError):
+    """更新失败异常。"""
+
+    pass

dataops_sdk-0.3.0/src/dataops_sdk/runner/__init__.py

@@ -0,0 +1,46 @@
+"""Runner 包 - 独立的脚本执行工具集。
+
+提供脚本执行所需的数据库连接、DuckDB 管理、Neo4j 连接等工具。
+可作为独立模块被导入使用。
+
+Modules:
+    schemas: Nacos 配置数据模型
+    utils: 通用工具（数据库连接、加密、日志等）
+
+Example:
+    >>> from runner.utils import DatabaseConn, DuckDBManager
+"""
+
+from .datasource_factory import get_datasource_instent
+from .schemas.nacos import NacosSettings
+from .utils import (
+    DatabaseConn,
+    FTPFileTransfer,
+    KafkaQueue,
+    Neo4jConn,
+    SFTPFileTransfer,
+    decrypt_body,
+    encrypt_body,
+    generate_uuid,
+    get_service_instance,
+    getLogger,
+    parse_args,
+    print_json,
+)
+
+__all__ = [
+    "DatabaseConn",
+    "FTPFileTransfer",
+    "KafkaQueue",
+    "SFTPFileTransfer",
+    "Neo4jConn",
+    "decrypt_body",
+    "encrypt_body",
+    "getLogger",
+    "print_json",
+    "generate_uuid",
+    "parse_args",
+    "NacosSettings",
+    "get_service_instance",
+    "get_datasource_instent",
+]

dataops_sdk-0.3.0/src/dataops_sdk/runner/datasource_factory.py

@@ -0,0 +1,83 @@
+"""
+数据源实例工厂模块。
+
+根据数据源元数据动态创建对应的连接实例（数据库或消息队列）。
+"""
+
+from dataops_sdk.services.datasource import service_get_datasource_by_name
+
+
+def get_datasource_instent(datasource: str, log_sql: bool | None = None):
+    """
+    根据数据源名称获取对应的连接实例。
+
+    通过数据源名称从平台查询元数据，根据数据源的 category（数据库/消息队列）
+    和 type（具体类型）动态创建并返回相应的连接实例。
+
+    Args:
+        datasource: 数据源名称，用于从平台查询数据源元数据。
+        log_sql: 是否记录 SQL 执行日志，传递给数据库连接实例。
+
+    Returns:
+        DatabaseConn: 通用数据库连接实例（MySQL、PostgreSQL 等）。
+        Neo4jConn: Neo4j 图数据库连接实例。
+        KafkaQueue: Kafka 消息队列连接实例。
+
+    Raises:
+        NotImplementedError: 当数据源类型未被支持时抛出。
+
+    Note:
+        当前支持的 category:
+        - database: 关系型/图数据库（Neo4j、MySQL、PostgreSQL 等）
+        - queue: 消息队列（Kafka）
+    """
+    # 从平台查询数据源元数据
+    meta = service_get_datasource_by_name(datasource)
+
+    # 根据数据源类别分发处理
+    if meta.data.category == "queue":
+        # 消息队列类型处理
+        if meta.data.type == "kafka":
+            # Kafka 连接需要 bootstrap_servers
+            assert meta.data.bootstrap_servers, "bootstrap_servers is required, but not provided"
+            from dataops_sdk.runner import KafkaQueue
+
+            return KafkaQueue(meta.data.bootstrap_servers)
+        else:
+            raise NotImplementedError(f"Queue type {meta.data.type} is not supported")
+
+    elif meta.data.category == "database":
+        # 数据库类型处理，验证必要的连接参数
+        assert meta.data.ip, "host is required, but not provided"
+        assert meta.data.port, "port is required, but not provided"
+        assert meta.data.username, "user is required, but not provided"
+        assert meta.data.password, "password is required, but not provided"
+        assert meta.data.database, "database is required, but not provided"
+
+        if meta.data.type == "neo4j":
+            # Neo4j 图数据库使用专用连接类
+            from dataops_sdk.runner import Neo4jConn
+
+            return Neo4jConn(
+                host=meta.data.ip,
+                port=meta.data.port,
+                user=meta.data.username,
+                password=meta.data.password,
+                db=meta.data.database,
+                log_sql=log_sql,
+                name=meta.title,
+            )
+        else:
+            # 通用数据库连接，支持 MySQL、PostgreSQL 等
+            from dataops_sdk.runner import DatabaseConn
+
+            return DatabaseConn(
+                host=meta.data.ip,
+                port=meta.data.port,
+                user=meta.data.username,
+                password=meta.data.password,
+                db=meta.data.database,
+                db_type=meta.data.type,
+                log_sql=log_sql,
+                name=meta.title,
+            )

dataops_sdk-0.3.0/src/dataops_sdk/runner/schemas/nacos/__init__.py

@@ -0,0 +1,5 @@
+from .config import NacosSettings
+from .instance import Instance
+from .service import Service
+
+__all__ = ["Service", "Instance", "NacosSettings"]

dataops_sdk-0.3.0/src/dataops_sdk/runner/schemas/nacos/config.py

@@ -0,0 +1,40 @@
+"""Nacos 配置模块。
+
+定义 Nacos 相关的配置数据模型，与主应用中的 Settings 类似，
+但用于 runner 子包独立运行时。
+
+Classes:
+    NacosSettings: Nacos 配置类
+
+Example:
+    >>> from runner.schemas.nacos import NacosSettings
+    >>> config = NacosSettings()
+"""
+
+from typing import Annotated
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class NacosSettings(BaseSettings):
+    """Nacos 配置类。
+
+    从 .env 文件和环境变量加载配置。配置项包括后端监听、
+    子系统连接和 Nacos 服务发现参数。
+
+    Attributes:
+        nacos_addrs: NACOS服务地址
+        nacos_user: NACOS用户名
+        nacos_password: NACOS密码
+        nacos_discovery_namespace: NACOS服务发现命名空间
+        nacos_group_name: NACOS服务发现组名
+    """
+
+    nacos_addrs: Annotated[str, Field(description="NACOS服务地址")]
+    nacos_user: Annotated[str, Field(description="NACOS用户名")]
+    nacos_password: Annotated[str, Field(description="NACOS密码")]
+    nacos_discovery_namespace: Annotated[str, Field(description="NACOS服务发现命名空间")]
+    nacos_group_name: Annotated[str, Field(description="NACOS服务发现组名")] = "DEFAULT_GROUP"
+
+    model_config = SettingsConfigDict(extra="ignore")

dataops_sdk-0.3.0/src/dataops_sdk/runner/schemas/nacos/constants.py

@@ -0,0 +1,235 @@
+class Constants:
+    NAMING_MODULE = "naming"
+
+    CONFIG_MODULE = "config"
+
+    AI_MODULE = "ai"
+
+    LABEL_SOURCE = "source"
+
+    LABEL_SOURCE_SDK = "sdk"
+
+    LABEL_SOURCE_CLUSTER = "cluster"
+
+    LABEL_MODULE = "module"
+
+    CLIENT_VERSION = "Nacos-Python-Client:v3.2.0"
+
+    DATA_IN_BODY_VERSION = 204
+
+    DEFAULT_GROUP = "DEFAULT_GROUP"
+
+    WEB_CONTEXT = "/nacos"
+
+    APPNAME = "AppName"
+
+    UNKNOWN_APP = "UnknownApp"
+
+    DEFAULT_DOMAINNAME = "commonconfig.config-host.taobao.com"
+
+    DAILY_DOMAINNAME = "commonconfig.taobao.net"
+
+    NULL = ""
+
+    DATAID = "dataId"
+
+    GROUP = "group"
+
+    DEFAULT_HEARTBEAT_INTERVAL = 5
+
+    LAST_MODIFIED = "Last-Modified"
+
+    ACCEPT_ENCODING = "Accept-Encoding"
+
+    CONTENT_ENCODING = "Content-Encoding"
+
+    PROBE_MODIFY_REQUEST = "Listening-Configs"
+
+    PROBE_MODIFY_RESPONSE = "Probe-Modify-Response"
+
+    PROBE_MODIFY_RESPONSE_NEW = "Probe-Modify-Response-New"
+
+    USE_ZIP = "true"
+
+    CONTENT_MD5 = "Content-MD5"
+
+    CONFIG_VERSION = "Config-Version"
+
+    CONFIG_TYPE = "Config-Type"
+
+    ENCRYPTED_DATA_KEY = "Encrypted-Data-Key"
+
+    IF_MODIFIED_SINCE = "If-Modified-Since"
+
+    SPACING_INTERVAL = "client-spacing-interval"
+
+    BASE_PATH = "/v1/cs"
+
+    SERVICE_BASE_PATH = "/v1/ns"
+
+    CONFIG_CONTROLLER_PATH = BASE_PATH + "/configs"
+
+    TOKEN = "token"
+
+    ACCESS_TOKEN = "accessToken"
+
+    TOKEN_TTL = "tokenTtl"
+
+    GLOBAL_ADMIN = "globalAdmin"
+
+    USERNAME = "username"
+
+    TOKEN_REFRESH_WINDOW = "tokenRefreshWindow"
+
+    # second.
+    ASYNC_UPDATE_ADDRESS_INTERVAL = 300
+
+    # second.
+    POLLING_INTERVAL_TIME = 15
+
+    # millisecond.
+    ONCE_TIMEOUT = 2000
+
+    # millisecond.
+    SO_TIMEOUT = 60000
+
+    # millisecond.
+    CONFIG_LONG_POLL_TIMEOUT = 30000
+
+    # millisecond.
+    MIN_CONFIG_LONG_POLL_TIMEOUT = 10000
+
+    # millisecond.
+    CONFIG_RETRY_TIME = 2000
+
+    # Maximum number of retries.
+    MAX_RETRY = 3
+
+    # millisecond.
+    RECV_WAIT_TIMEOUT = ONCE_TIMEOUT * 5
+
+    ENCODE = "UTF-8"
+
+    MAP_FILE = "map-file.js"
+
+    FLOW_CONTROL_THRESHOLD = 20
+
+    FLOW_CONTROL_SLOT = 10
+
+    FLOW_CONTROL_INTERVAL = 1000
+
+    DEFAULT_PROTECT_THRESHOLD = 0.0
+
+    LINE_SEPARATOR = chr(1)
+
+    WORD_SEPARATOR = chr(2)
+
+    LONGPOLLING_LINE_SEPARATOR = "\r\n"
+
+    CLIENT_APPNAME_HEADER = "Client-AppName"
+
+    APP_NAME_HEADER = "AppName"
+
+    CLIENT_REQUEST_TS_HEADER = "Client-RequestTS"
+
+    CLIENT_REQUEST_TOKEN_HEADER = "Client-RequestToken"
+
+    ATOMIC_MAX_SIZE = 1000
+
+    NAMING_INSTANCE_ID_SPLITTER = "#"
+
+    NAMING_INSTANCE_ID_SEG_COUNT = 4
+
+    NAMING_HTTP_HEADER_SPLITTER = "\\|"
+
+    DEFAULT_CLUSTER_NAME = "DEFAULT"
+
+    DEFAULT_HEART_BEAT_TIMEOUT = 15000
+
+    DEFAULT_IP_DELETE_TIMEOUT = 30000
+
+    DEFAULT_HEART_BEAT_INTERVAL = 5000
+
+    DEFAULT_NAMESPACE_ID = ""
+
+    NAMING_DEFAULT_NAMESPACE_ID = "public"
+
+    DEFAULT_USE_CLOUD_NAMESPACE_PARSING = True
+
+    WRITE_REDIRECT_CODE = 307
+
+    RESPONSE_SUCCESS_CODE = 200
+
+    REQUEST_DOMAIN_RETRY_TIME = 3
+
+    SERVICE_INFO_SPLITER = "@@"
+
+    CONFIG_INFO_SPLITER = "@@"
+
+    SERVICE_INFO_SPLIT_COUNT = 2
+
+    NULL_STRING = "null"
+
+    NUMBER_PATTERN = "^\\d+$"
+
+    ANY_PATTERN = ".*"
+
+    DEFAULT_INSTANCE_ID_GENERATOR = "simple"
+
+    SNOWFLAKE_INSTANCE_ID_GENERATOR = "snowflake"
+
+    HTTP_PREFIX = "http"
+
+    ALL_PATTERN = "*"
+
+    COLON = ":"
+
+    LINE_BREAK = "\n"
+
+    POUND = "#"
+
+    VIPSERVER_TAG = "Vipserver-Tag"
+
+    AMORY_TAG = "Amory-Tag"
+
+    LOCATION_TAG = "Location-Tag"
+
+    CHARSET_KEY = "charset"
+
+    EX_CONFIG_INFO = "exConfigInfo"
+
+    GROUP_NAME_KEY = "groupName"
+
+    SERVICE_NAME_KEY = "serviceName"
+
+    CIPHER_PRE_FIX = "cipher-"
+
+    DEFAULT_PORT = 8848
+
+    GRPC_MAX_RECEIVE_MESSAGE_LENGTH = 100 * 1024 * 1024
+
+    GRPC_KEEPALIVE_TIME_MILLS = 60 * 1000
+
+    GRPC_INITIAL_WINDOW_SIZE = 10 * 1024 * 1024
+
+    GRPC_INITIAL_CONN_WINDOW_SIZE = 10 * 1024 * 1024
+
+    KEEP_ALIVE_TIME_MILLS = 5000
+
+    INTERNAL_TIME_MILLS = 3000
+
+    DEFAULT_GRPC_TIMEOUT_MILLS = 3000
+
+    GRPC_CAPABILITY_NEGOTIATION_TIMEOUT = 5000
+
+    GRPC_PORT_OFFSET = 1000  # gRPC port = HTTP port + offset
+
+    DEFAULT_TIMEOUT_MILLS = 10000
+
+    PER_TASK_CONFIG_SIZE = 3000
+
+    MSE_KMS_V1_DEFAULT_KEY_ID = "alias/acs/mse"
+
+    KMS_AES_128_ALGORITHM_NAME = "AES_128"
+
+    KMS_AES_256_ALGORITHM_NAME = "AES_256"