dataops-sdk 0.3.0__py3-none-any.whl

dataops_sdk/services/skill.py

@@ -0,0 +1,276 @@
+from collections.abc import Mapping
+from pathlib import Path
+
+from pydantic_core import to_json
+
+from dataops_sdk.exceptions import AuthenticationError, DataOpsError
+from dataops_sdk.schemas import DataSkill, MetaRelease, MetaSkill, MetaType, SkillCallResult
+from dataops_sdk.services.meta import (
+    release_list_get_by_code,
+    release_list_get_by_name,
+    release_list_list,
+)
+from dataops_sdk.session import get_session, post_multipart
+from dataops_sdk.utils import encrypt_body, filter_nodes_by_pattern
+
+__all__ = [
+    "service_call_skill_by_code",
+    "service_delete_skill",
+    "service_delete_sub_file",
+    "service_get_skill_by_code",
+    "service_get_skill_by_name",
+    "service_import_skill",
+    "service_list_skill_files",
+    "service_list_skills",
+    "service_upload_sub_file",
+]
+
+
+def service_list_skills(query: str | None = None) -> list[MetaRelease]:
+    """列出技能（可选正则过滤）.
+
+    Args:
+        query: 正则表达式，匹配 NAME / TITLE / REMARK / DIGEST.
+
+    Returns:
+        MetaRelease 对象列表
+
+    Raises:
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证或认证失败时
+    """
+    nodes = release_list_list(MetaType.SKILL)
+    if query:
+        nodes = filter_nodes_by_pattern(nodes, query, include_digest=True)
+    return nodes
+
+
+def service_get_skill_by_code(code: str) -> MetaSkill:
+    """按编码获取技能详情.
+
+    Args:
+        code: 技能编码.
+
+    Returns:
+        MetaSkill 实例（含 data 字段，已自动从 rawData 反序列化）
+
+    Raises:
+        NotFoundError: 找不到对应技能时
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    return release_list_get_by_code(MetaType.SKILL, code, _class=DataSkill)
+
+
+def service_get_skill_by_name(name: str) -> MetaSkill:
+    """按业务名获取技能详情.
+
+    Args:
+        name: 技能业务名.
+
+    Returns:
+        MetaSkill 实例
+
+    Raises:
+        NotFoundError: 找不到对应技能时
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    return release_list_get_by_name(MetaType.SKILL, name, _class=DataSkill)
+
+
+def service_list_skill_files(skill_id: str) -> list[MetaRelease]:
+    """列出指定技能下的所有子文件（type=skill_file, parent_id=skill_id）.
+
+    Args:
+        skill_id: 技能的 meta ID（不是 code）.
+
+    Returns:
+        MetaRelease 对象列表（每个对应一个 skill_file 子文件）
+
+    Raises:
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证或认证失败时
+    """
+    return release_list_list(
+        MetaType.SKILL_FILE,
+        extra_conditions=[{"name": "parent_id", "value": skill_id, "alias": "mrl"}],
+    )
+
+
+def _post_skill(url: str, body: dict | None = None, *, params: dict | None = None) -> dict:
+    """通用 skill POST 请求，返回 ``resp_body['data']``.
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    try:
+        with get_session() as session:
+            kwargs: dict = {}
+            if body is not None:
+                kwargs["content"] = to_json(encrypt_body(body))
+            if params is not None:
+                kwargs["params"] = params
+            resp = session.post(url, **kwargs)
+            resp_body = resp.json()
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                raise DataOpsError(
+                    resp_body.get("msg", f"HTTP {resp.status_code}"), resp.status_code
+                )
+        return resp_body.get("data") or {}
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+
+
+def service_import_skill(
+    file_path: str | Path,
+    sub_type: str = "llm",
+    name: str | None = None,
+    title: str | None = None,
+    ref_node_id: str | None = None,
+    ref_type: str | None = None,
+) -> dict:
+    """从 ZIP 包导入技能（multipart 上传）.
+
+    Args:
+        file_path: 本地 ZIP 或 .md 文件路径.
+        sub_type: 技能子类型，可选 ``llm`` / ``api`` / ``report``，默认 ``llm``.
+        name: 可选覆盖 SKILL.md frontmatter 中的 ``name`` 字段.
+        title: 可选覆盖 SKILL.md 中的 ``title`` 字段.
+        ref_node_id: 分组节点 ID；指定后导入的技能会挂到该分组下.
+        ref_type: 相对位置 ``before`` / ``after``，仅在 ``ref_node_id`` 非空时生效.
+
+    Returns:
+        后端返回的 ``data`` 字典（通常为 ``GroupTree``）.
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+        FileNotFoundError: 本地文件不存在时
+    """
+    path = Path(file_path)
+    if not path.exists():
+        raise FileNotFoundError(f"文件不存在: {path}")
+
+    form_data: dict[str, str] = {"subType": sub_type}
+    if name:
+        form_data["name"] = name
+    if title:
+        form_data["title"] = title
+    if ref_node_id:
+        form_data["refNodeId"] = ref_node_id
+    if ref_type:
+        form_data["refType"] = ref_type
+
+    with (
+        open(path, "rb") as f,
+        post_multipart(
+            "/data/skill/import",
+            files={"file": (path.name, f)},
+            data=form_data,
+        ) as resp,
+    ):
+        body = resp.json()
+        return body.get("data") or {}
+
+
+def service_delete_skill(name: str) -> dict:
+    """按业务名删除技能（级联删除所有 skill_file）.
+
+    Args:
+        name: 技能业务名（注意不是 code）.
+
+    Returns:
+        后端返回的 ``data`` 字典（通常为 ``GroupTree``）.
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    return _post_skill(f"/data/skill/delete/{name}")
+
+
+def service_upload_sub_file(
+    skill_name: str,
+    file_path: str | Path,
+    path: str,
+) -> None:
+    """为已存在技能上传子文件（doc / code / image 三类之一）.
+
+    Args:
+        skill_name: 目标技能的业务名.
+        file_path: 本地文件路径.
+        path: 在技能包内的相对路径，如 ``docs/readme.md`` / ``script/main.py``.
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+        FileNotFoundError: 本地文件不存在时
+    """
+    fp = Path(file_path)
+    if not fp.exists():
+        raise FileNotFoundError(f"文件不存在: {fp}")
+
+    with (
+        open(fp, "rb") as f,
+        post_multipart(
+            f"/data/skill/{skill_name}/subfile",
+            files={"file": (fp.name, f)},
+            data={"path": path},
+        ) as resp,
+    ):
+        resp.json()  # 仅校验响应
+
+
+def service_delete_sub_file(skill_name: str, path: str) -> None:
+    """删除技能下的指定子文件.
+
+    Args:
+        skill_name: 目标技能的业务名.
+        path: 子文件相对路径（与上传时一致）.
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    _post_skill(f"/data/skill/{skill_name}/subfile/delete", params={"path": path})
+
+
+def service_call_skill_by_code(code: str, inputs: Mapping) -> SkillCallResult:
+    """调用 skill 同步执行，返回 Python 容器运行结果.
+
+    对应后端 ``POST /data/skill/callByCode/{code}``，入参 ``{"inputs": {...}}``。
+
+    Args:
+        code: 技能编码.
+        inputs: 传给 Python 容器的入参 dict.
+
+    Returns:
+        SkillCallResult 实例，含 ``inst_id`` / ``status`` / ``output_data`` / ``error_message``.
+
+    Raises:
+        DataOpsError: API 返回错误或响应 data 非 dict 时
+        AuthenticationError: 未认证或认证失败时
+    """
+    try:
+        with get_session() as session:
+            resp = session.post(
+                f"/data/skill/callByCode/{code}",
+                content=to_json(encrypt_body({"inputs": dict(inputs)})),
+            )
+            resp_body = resp.json()
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                raise DataOpsError(
+                    resp_body.get("msg", f"HTTP {resp.status_code}"), resp.status_code
+                )
+            data = resp_body.get("data")
+            if not isinstance(data, dict):
+                raise DataOpsError(f"返回 data 格式异常: {type(data).__name__}")
+            return SkillCallResult.model_validate(data)
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e

dataops_sdk/services/table.py

@@ -0,0 +1,340 @@
+import re
+from collections.abc import Mapping
+from typing import Any
+
+from pydantic import ConfigDict, JsonValue, TypeAdapter
+from pydantic_core import to_json
+
+from dataops_sdk.exceptions import DataOpsError
+from dataops_sdk.schemas import (
+    DataDataBase,
+    Meta,
+    MetaRelease,
+    MetaType,
+    PageParam,
+    SqlQueryParam,
+)
+from dataops_sdk.schemas.enums import DataSourceType
+from dataops_sdk.services.datasource import service_get_datasource_by_name
+from dataops_sdk.services.meta import (
+    release_list_get_by_code,
+    release_list_get_by_name,
+    release_list_list,
+)
+from dataops_sdk.session import get_session
+from dataops_sdk.utils import encrypt_body
+
+# 允许的读操作 SQL 关键字前缀
+_READ_ONLY_PREFIXES = ("select", "show", "describe", "explain", "use")
+
+
+def validate_read_only_sql(sql: str) -> None:
+    """校验 SQL 是否为只读操作.
+
+    Raises:
+        ValueError: SQL 为修改操作时
+    """
+    normalized = sql.lower().strip().strip("(")
+    global _READ_ONLY_PREFIXES
+    if not normalized.startswith(_READ_ONLY_PREFIXES):
+        raise ValueError(f"仅支持 {'/'.join(_READ_ONLY_PREFIXES)} 语句")
+
+
+# LIMIT/OFFSET 模式（不区分大小写）
+# 匹配 LIMIT n [OFFSET n]、LIMIT n, n、LIMIT n OFFSET n 形式
+_LIMIT_PATTERN = re.compile(
+    r"\s+LIMIT\s+(?:\d+|\?)\s*(?:,\s*(?:\d+|\?))?\s*(?:OFFSET\s+(?:\d+|\?))?",
+    re.IGNORECASE,
+)
+_OFFSET_PATTERN = re.compile(
+    r"\s+OFFSET\s+(?:\d+|\?)\s*(?:ROWS)?",
+    re.IGNORECASE,
+)
+# Oracle FETCH 模式
+_ORACLE_FETCH_PATTERN = re.compile(
+    r"\s+OFFSET\s+(?:\d+|\?)\s+ROWS\s+FETCH\s+NEXT\s+(?:\d+|\?)\s+ROWS\s+ONLY",
+    re.IGNORECASE,
+)
+
+
+def remove_limit_offset(sql: str, dialect: DataSourceType) -> str:
+    """移除 SQL 中的 LIMIT/OFFSET 子句.
+
+    Args:
+        sql: 原始 SQL
+        dialect: 数据库类型
+
+    Returns:
+        移除 LIMIT/OFFSET 后的 SQL
+    """
+    if dialect == DataSourceType.ORACLE:
+        # Oracle: 移除 OFFSET ... ROWS FETCH NEXT ... ROWS ONLY
+        sql = _ORACLE_FETCH_PATTERN.sub("", sql)
+    elif dialect in {
+        DataSourceType.MYSQL,
+        DataSourceType.POSTGRESQL,
+        DataSourceType.DM,
+        DataSourceType.DORIS,
+    }:
+        # MySQL/PostgreSQL/DM/DORIS: 分别移除 LIMIT 和 OFFSET
+        sql = _LIMIT_PATTERN.sub("", sql)
+        sql = _OFFSET_PATTERN.sub("", sql)
+    else:
+        raise NotImplementedError(f"不支持 {dialect} 数据库类型")
+    return sql.strip()
+
+
+def service_list_tables(datasource: str) -> list[MetaRelease]:
+    """搜索数据源中的表.
+
+    Args:
+        datasource: 数据源名称
+
+    Returns:
+        MetaRelease 对象列表
+
+    Raises:
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证时
+    """
+    headers = {"x-sub-system": datasource}
+    filters = (SqlQueryParam().add_eq("sub_system", datasource, "mrl"),)
+    return release_list_list(MetaType.TABLE, filters=filters, headers=headers)
+
+
+def service_count_table_by_code(
+    code: str,
+    query_param: SqlQueryParam | None = None,
+    datasource: str | None = None,
+) -> int:
+    """获取表数据数量.
+
+    Args:
+        code: 表编码
+        query_param: 查询条件
+
+    Returns:
+        表数据数量
+
+    Raises:
+        NotFoundError: 表不存在时
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证时
+    """
+    url = "/data/table/countData"
+    if query_param is None:
+        query_param = SqlQueryParam()
+
+    if not datasource:
+        meta = release_list_get_by_code(MetaType.TABLE, code, _class=DataDataBase)
+        datasource = meta.subSystem
+
+    body = {
+        "subSystem": datasource,
+        "code": code,
+        "type": MetaType.TABLE,
+        "queryParam": query_param,
+    }
+
+    with get_session() as session:
+        resp = session.post(url, content=to_json(encrypt_body(body)))
+        resp_body = resp.json()
+    if resp.status_code != 200:
+        raise DataOpsError(f"查询失败，{resp_body.get('msg')}", resp.status_code)
+    if resp_body.get("data") is None:
+        return 0
+    return resp_body["data"]
+
+
+def service_count_table_by_name(
+    datasource: str,
+    name: str,
+    query_param: SqlQueryParam | None = None,
+) -> int:
+    """通过数据源 + 表名获取表数据数量.
+
+    Args:
+        datasource: 数据源名称
+        name: 表名
+        query_param: 查询条件
+
+    Returns:
+        表数据数量
+
+    Raises:
+        NotFoundError: 表不存在时
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证时
+    """
+    headers = {"x-sub-system": datasource}
+    meta = release_list_get_by_name(MetaType.TABLE, name, headers=headers)
+    return service_count_table_by_code(meta.code, query_param, datasource=datasource)
+
+
+def service_get_table_data_by_code(
+    code: str,
+    page_param: PageParam | None = None,
+    query_param: SqlQueryParam | None = None,
+    datasource: str | None = None,
+) -> list[Mapping[str, JsonValue]]:
+    """查询表数据.
+
+    Args:
+        code: 表编码
+        page_param: 分页参数
+        query_param: 查询条件
+
+    Returns:
+        表数据行列表（每行为字典）
+
+    Raises:
+        NotFoundError: 表不存在时
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证时
+    """
+    url = "/data/table/findData"
+    if page_param is None:
+        page_param = PageParam()
+    if query_param is None:
+        query_param = SqlQueryParam()
+
+    if not datasource:
+        meta = release_list_get_by_code(MetaType.TABLE, code, _class=DataDataBase)
+        datasource = meta.subSystem
+
+    body = {
+        "subSystem": datasource,
+        "code": code,
+        "pageParam": page_param,
+        "queryParam": query_param,
+    }
+
+    with get_session(timeout=15) as session:
+        resp = session.post(url, content=to_json(encrypt_body(body)))
+        resp_body = resp.json()
+    if resp.status_code != 200:
+        raise DataOpsError(f"查询失败，{resp_body.get('msg')}", resp.status_code)
+    adapter = TypeAdapter(list[Mapping[str, JsonValue]], config=ConfigDict(str_to_lower=True))
+
+    return adapter.validate_python(resp_body.get("data", []))
+
+
+def service_get_table_data_by_name(
+    datasource: str,
+    name: str,
+    page_param: PageParam | None = None,
+    query_param: SqlQueryParam | None = None,
+) -> list[dict[str, Any]]:
+    headers = {"x-sub-system": datasource}
+    meta = release_list_get_by_name(MetaType.TABLE, name, headers=headers, _class=DataDataBase)
+    return service_get_table_data_by_code(meta.code, page_param, query_param, datasource)
+
+
+def service_get_table_detail_by_name(
+    datasource: str,
+    name: str,
+) -> Meta[DataDataBase]:
+    """指定数据源表详情.
+
+    Args:
+        datasource: 数据源名称
+        name: 表名
+
+    Returns:
+        数据源 DDL 模型
+
+    Raises:
+        ValueError: 无效类型
+        NotFoundError: 数据源不存在
+        DataOpsError: API 请求失败
+        AuthenticationError: 未认证
+    """
+    headers = {"x-sub-system": datasource}
+    meta = release_list_get_by_name(MetaType.TABLE, name, headers=headers, _class=DataDataBase)
+    return meta
+
+
+def service_get_table_detail_by_code(code: str) -> Meta[DataDataBase]:
+    """指定数据源表详情.
+
+    Args:
+        code: 表编码
+
+    Returns:
+        数据源 DDL 模型
+
+    Raises:
+        ValueError: 无效类型
+        NotFoundError: 数据源不存在
+        DataOpsError: API 请求失败
+        AuthenticationError: 未认证
+    """
+    meta = release_list_get_by_code(MetaType.TABLE, code, _class=DataDataBase)
+    return meta
+
+
+def service_exec_sql_by_datasource(
+    datasource: str,
+    sql: str,
+    page_num: int = 0,
+    page_size: int = 20,
+) -> list[Mapping[str, JsonValue]]:
+    """执行任意 SQL 查询.
+
+    Args:
+        datasource: 数据源名称
+        sql: SQL 查询语句
+        page_num: 页码 [default: 0]
+        page_size: 每页大小 [default: 20]
+
+    Returns:
+        查询结果行列表（每行为字典）
+
+    Raises:
+        NotFoundError: 表不存在时
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证时
+        ValueError: SQL 为修改操作时
+    """
+    if page_num < 0:
+        raise ValueError("page_num must >= 0")
+    if page_size <= 0:
+        raise ValueError("page_size must > 0")
+    validate_read_only_sql(sql)
+    meta = service_get_datasource_by_name(datasource)
+    sql = paginate_query(meta.data.type, sql, page_size, page_num * page_size)
+    # Lazy import：依赖 dataops_sdk[runner] extra，仅在调用本函数时才需要安装
+    from dataops_sdk.runner import DatabaseConn
+
+    with DatabaseConn(datasource, log_sql=False) as conn:
+        results = conn.select(sql, size=page_size)
+    return results
+
+
+def paginate_query(dialect: DataSourceType, base_sql: str, limit: int, offset: int):
+    """通用的分页函数"""
+    # 不同方言的分页包装
+    if dialect in {
+        DataSourceType.MYSQL,
+        DataSourceType.POSTGRESQL,
+        DataSourceType.DM,
+        DataSourceType.DORIS,
+    }:
+        # 检测 + 移除 原来有的 LIMIT/OFFSET
+        base_sql = remove_limit_offset(base_sql, dialect)
+
+        # 修改 SQL
+        sql = f"{base_sql} LIMIT {limit} OFFSET {offset}"
+    elif dialect == DataSourceType.ORACLE:
+        # 检测 + 移除 原来有的 LIMIT/OFFSET
+        base_sql = remove_limit_offset(base_sql, dialect)
+
+        # 修改 SQL（内层子查询加别名避免 ORA-00918）
+        sql = f"""
+            SELECT * FROM (
+                {base_sql}
+            ) _t OFFSET {offset} ROWS FETCH NEXT {limit} ROWS ONLY
+        """
+    else:
+        raise ValueError(f"Unsupported database: {dialect}")
+    return sql