dataops-sdk 0.3.0__py3-none-any.whl

dataops_sdk/services/meta.py

@@ -0,0 +1,523 @@
+"""统一的 Meta 操作模块（服务层）.
+
+提供对 meta release 列表查询和单个获取的通用功能.
+本模块为纯服务层，不依赖 typer/Rich，可供 Python 代码直接调用。
+CLI 命令请使用 meta_cli 模块。
+"""
+
+from collections.abc import Mapping, Sequence
+
+from pydantic import ConfigDict, JsonValue, TypeAdapter
+from pydantic_core import to_json, to_jsonable_python
+
+from dataops_sdk.exceptions import (
+    AuthenticationError,
+    DataOpsError,
+    FailedToCreateError,
+    NotFoundError,
+)
+from dataops_sdk.schemas import (
+    DEFAULT_META_DATA_TYPE,
+    META_DATA_TYPE,
+    InnerMetaType,
+    Meta,
+    MetaCreate,
+    MetaRef,
+    MetaRelease,
+    MetaType,
+    PageParam,
+    SqlQueryParam,
+)
+from dataops_sdk.session import get_session
+from dataops_sdk.utils import encrypt_body
+
+
+def release_list_list(
+    meta_type: MetaType | InnerMetaType | str,
+    filters: Sequence[dict] | None = None,
+    headers: dict[str, str] | None = None,
+    datasource: str | None = None,
+    extra_conditions: Sequence[dict] | None = None,
+) -> list[MetaRelease]:
+    """获取 Meta Release 列表.
+
+    Args:
+        meta_type: 元数据类型（MetaType 枚举或字符串）.
+        filters: SQL 查询过滤条件列表.
+        headers: 额外的请求头.
+        datasource: 数据源；非空时写入 ``x-sub-system`` 头.
+        extra_conditions: 在默认 ``type=meta_type`` 之上追加等值条件列表，每项形如
+            ``{"name": "parent_id", "value": "xxx", "alias": "mrl"}``. 主要用于按
+            parent_id 等辅助字段过滤。
+
+    Returns:
+        MetaRelease 对象列表
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    type_str = meta_type.value if isinstance(meta_type, (MetaType, InnerMetaType)) else meta_type
+    url = "/meta/release/list/list"
+
+    sql_query_param = SqlQueryParam().add_eq("type", type_str, "mrl")
+    for cond in extra_conditions or []:
+        sql_query_param.add_eq(cond["name"], cond["value"], cond.get("alias", "mrl"))
+
+    headers = headers or {}
+    if datasource:
+        headers["x-sub-system"] = datasource
+
+    body = {
+        "filters": to_jsonable_python(filters or []),
+        "pageParam": PageParam(limit=-1),
+        "queryParam": sql_query_param,
+    }
+
+    try:
+        with get_session() as session:
+            resp = session.post(url, headers=headers, content=to_json(encrypt_body(body)))
+            resp_body = resp.json()
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                raise DataOpsError(
+                    resp_body.get("msg", f"HTTP {resp.status_code}"), resp.status_code
+                )
+            group_nodes = TypeAdapter(
+                list[MetaRelease], config=ConfigDict(str_to_lower=True)
+            ).validate_python(resp_body["data"])
+        return group_nodes
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+
+
+def release_list_page(
+    meta_type: MetaType | InnerMetaType | str,
+    filters: Sequence[dict] | None = None,
+    headers: dict[str, str] | None = None,
+    datasource: str | None = None,
+    page: int = 0,
+    size: int = 20,
+):
+    """分页获取 Meta Release 列表.
+
+    Args:
+        type: 类型
+        filters: SQL 查询过滤条件列表
+        headers: 额外的请求头
+        datasource: 数据源
+        page: 页码
+        size: 每页大小
+
+    Returns:
+        list[MetaRelease] 对象列表
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    type_str = meta_type.value if isinstance(meta_type, (MetaType, InnerMetaType)) else meta_type
+    url = "/meta/release/list/page"
+
+    sql_query_param = SqlQueryParam().add_eq("type", type_str, "mrl")
+
+    if page < 0:
+        raise ValueError("page must be greater than or equal to 0")
+
+    headers = headers or {}
+    if datasource:
+        headers["x-sub-system"] = datasource
+
+    body = {
+        "filters": to_jsonable_python(filters or []),
+        "pageParam": PageParam(start=page * size, limit=size),
+        "queryParam": sql_query_param,
+    }
+
+    try:
+        with get_session() as session:
+            resp = session.post(url, headers=headers, content=to_json(encrypt_body(body)))
+            resp_body = resp.json()
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                raise DataOpsError(
+                    resp_body.get("msg", f"HTTP {resp.status_code}"), resp.status_code
+                )
+            # TODO 修改
+            total = resp_body["data"]["total"]
+            page_num = resp_body["data"]["pageNum"]
+            page_size = resp_body["data"]["pageSize"]
+            group_nodes = TypeAdapter(
+                list[MetaRelease], config=ConfigDict(str_to_lower=True)
+            ).validate_python(resp_body["data"]["records"])
+        return group_nodes
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+
+
+def release_list_total(
+    meta_type: MetaType | InnerMetaType | str,
+    filters: Sequence[dict] | None = None,
+    headers: dict[str, str] | None = None,
+    datasource: str | None = None,
+) -> int:
+    """获取 Meta Release 列表总数.
+
+    Args:
+        meta_type: 元数据类型
+        filters: SQL 查询过滤条件列表
+        headers: 额外的请求头
+        datasource: 数据源
+
+    Returns:
+        int 总数
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    type_str = meta_type.value if isinstance(meta_type, (MetaType, InnerMetaType)) else meta_type
+    url = "/meta/release/list/page"
+
+    sql_query_param = SqlQueryParam().add_eq("type", type_str, "mrl")
+
+    headers = headers or {}
+    if datasource:
+        headers["x-sub-system"] = datasource
+
+    body = {
+        "filters": to_jsonable_python(filters or []),
+        "queryParam": sql_query_param,
+    }
+
+    try:
+        with get_session() as session:
+            resp = session.post(url, headers=headers, content=to_json(encrypt_body(body)))
+            resp_body = resp.json()
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                raise DataOpsError(
+                    resp_body.get("msg", f"HTTP {resp.status_code}"), resp.status_code
+                )
+        return resp_body["data"]
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+
+
+def release_list_get_by_code(
+    meta_type: MetaType | InnerMetaType | str,
+    code: str,
+    headers: dict[str, str] | None = None,
+    *,
+    _class: type[META_DATA_TYPE] = DEFAULT_META_DATA_TYPE,
+) -> Meta[META_DATA_TYPE]:
+    """根据编码获取 Meta Release 数据.
+
+    Args:
+        meta_type: 元数据类型字符串
+        code: 元数据编码
+        headers: 额外的请求头
+
+    Returns:
+        包含元数据的字典
+
+    Raises:
+        NotFoundError: 找不到对应元数据时
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    type_str = meta_type.value if isinstance(meta_type, (MetaType, InnerMetaType)) else meta_type
+    url = f"/meta/release/list/getByCode/{type_str}/{code}"
+
+    try:
+        with get_session() as session:
+            resp = session.get(url, headers=headers)
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                msg = resp.json().get("msg", f"HTTP {resp.status_code}")
+                raise DataOpsError(
+                    f"fail to get {type_str} by {code}, cause by {msg}", resp.status_code
+                )
+            resp_body = resp.json()
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+
+    if resp_body.get("data") is None:
+        raise NotFoundError(f"无法找到 {type_str} {code} 的信息")
+
+    raw_meta = resp_body["data"]
+    if not isinstance(raw_meta, dict):
+        raise DataOpsError(f"返回的数据格式不正确, 预期为字典, 实际为 {type(raw_meta)}")
+
+    return Meta[_class].model_validate(raw_meta)
+
+
+def release_list_get_by_name(
+    meta_type: MetaType | InnerMetaType | str,
+    name: str,
+    headers: dict[str, str] | None = None,
+    *,
+    _class: type[META_DATA_TYPE] = DEFAULT_META_DATA_TYPE,
+    datasource: str | None = None,
+) -> Meta[META_DATA_TYPE]:
+    """根据名称获取 Meta Release 数据.
+
+    Args:
+        meta_type: 元数据类型字符串
+        name: 元数据名称
+        headers: 额外的请求头
+
+    Returns:
+        包含元数据的字典
+
+    Raises:
+        NotFoundError: 找不到对应元数据时
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    type_str = meta_type.value if isinstance(meta_type, (MetaType, InnerMetaType)) else meta_type
+    url = f"/meta/release/list/getByName/{type_str}/{name}"
+
+    if datasource:
+        headers = headers or {}
+        headers["x-sub-system"] = datasource
+
+    try:
+        with get_session() as session:
+            resp = session.get(url, headers=headers)
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                msg = resp.json().get("msg", f"HTTP {resp.status_code}")
+                raise DataOpsError(
+                    f"fail to get {type_str} by {name}, cause by {msg}", resp.status_code
+                )
+            resp_body = resp.json()
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+
+    if resp_body.get("data") is None:
+        raise NotFoundError(f"无法找到 {type_str} {name} 的信息")
+
+    raw_meta = resp_body["data"]
+    if not isinstance(raw_meta, dict):
+        raise DataOpsError(f"返回的数据格式不正确, 预期为字典, 实际为 {type(raw_meta)}")
+
+    return Meta[_class].model_validate(raw_meta)
+
+
+def release_ref_list(
+    meta_type: MetaType | InnerMetaType | str,
+    code: str,
+    headers: dict[str, str] | None = None,
+) -> list[MetaRef]:
+    """获取 Meta Release 的正向引用列表（查询自己引用了谁）.
+
+    Args:
+        meta_type: 元数据类型 (如 "report", "table")
+        code: 元数据编码
+        headers: 额外的请求头
+
+    Returns:
+        list[MetaRef] 对象列表
+
+    Raises:
+        NotFoundError: 找不到对应元数据时
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    type_str = meta_type.value if isinstance(meta_type, (MetaType, InnerMetaType)) else meta_type
+    url = f"/meta/release/ref/list/{type_str}/{code}"
+
+    try:
+        with get_session() as session:
+            resp = session.get(url, headers=headers)
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                raise DataOpsError(f"HTTP {resp.status_code}", resp.status_code)
+            resp_body = resp.json()
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+
+    if resp_body.get("data") is None:
+        raise NotFoundError(f"无法找到 {meta_type} {code} 的信息")
+    results = TypeAdapter(list[MetaRef]).validate_python(resp_body["data"])
+    return results
+
+
+def release_ref_list_by_ref(
+    ref_type: MetaType | InnerMetaType | str,
+    ref_code: str,
+    headers: dict[str, str] | None = None,
+) -> list[MetaRef]:
+    """获取 Meta Release 的反向引用列表（查询谁引用了自己）.
+
+    Args:
+        ref_type: 引用元数据类型 (如 "report", "table")
+        ref_code: 引用元数据编码
+        headers: 额外的请求头
+        deduplicate: 是否去重
+
+    Returns:
+        list[MetaRef] 对象列表
+
+    Raises:
+        NotFoundError: 找不到对应元数据时
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    type_str = ref_type.value if isinstance(ref_type, (MetaType, InnerMetaType)) else ref_type
+    url = f"/meta/release/ref/listByRef/{type_str}/{ref_code}"
+
+    try:
+        with get_session() as session:
+            resp = session.get(url, headers=headers)
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                raise DataOpsError(f"HTTP {resp.status_code}", resp.status_code)
+            resp_body = resp.json()
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+
+    if resp_body.get("data") is None:
+        raise NotFoundError(f"无法找到 {ref_type} {ref_code} 的引用信息")
+    results = TypeAdapter(list[MetaRef]).validate_python(resp_body["data"])
+    return results
+
+
+def release_list_create(
+    meta: MetaCreate[META_DATA_TYPE],
+    params: Sequence[Mapping[str, JsonValue]] | None = None,
+    *,
+    headers: dict[str, str] | None = None,
+    _class: type[META_DATA_TYPE] = DEFAULT_META_DATA_TYPE,
+) -> Meta[META_DATA_TYPE]:
+    """创建 Meta 对象.
+
+    Args:
+        meta: 要创建的元数据对象
+        params: 可选的查询参数列表，用于模板渲染
+        headers: 额外的请求头
+        _class: 返回的 Meta 数据类类型，默认为 Meta
+
+    Returns:
+        创建成功的 Meta 对象
+
+    Raises:
+        FailedToCreateError: 创建失败时
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    body = {"meta": meta, "params": params}
+
+    try:
+        with get_session() as session:
+            resp = session.post(
+                "/meta/release/list/create",
+                headers=headers,
+                content=to_json(encrypt_body(body)),
+            )
+            resp_body = resp.json()
+            if resp.status_code == 401:
+                raise AuthenticationError("认证失败，请先登录")
+            if resp.status_code != 200:
+                raise DataOpsError(
+                    resp_body.get("msg", f"HTTP {resp.status_code}"), resp.status_code
+                )
+
+    except SystemExit as e:
+        raise AuthenticationError("认证失败，请先登录") from e
+    if resp_body.get("data") is None:
+        raise FailedToCreateError("创建失败")
+
+    raw_meta = resp_body["data"]
+    if not isinstance(raw_meta, dict):
+        raise DataOpsError(f"返回的数据格式不正确, 预期为字典, 实际为 {type(raw_meta)}")
+
+    return Meta[_class].model_validate(raw_meta)
+
+
+def release_list_update(
+    meta: Meta[META_DATA_TYPE],
+    # origin_meta: Meta[META_DATA_TYPE],
+    # meta: MetaUpdate[META_DATA_TYPE],
+    *,
+    headers: dict[str, str] | None = None,
+    datasource: str | None = None,
+    _class: type[META_DATA_TYPE] = DEFAULT_META_DATA_TYPE,
+):
+    """更新 Meta 对象.
+
+    Args:
+        meta: 元数据对象
+        headers: 额外的请求头
+        _class: 返回的 Meta 数据类类型，默认为 Meta
+
+    Returns:
+        更新后的 Meta 对象
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    """更新 Meta 对象.
+
+    Args:
+        origin_meta: 原始的元数据对象（更新前的完整对象）
+        meta: 要更新的元数据字段（支持部分更新，仅传入需要修改的字段）
+        headers: 额外的请求头
+        _class: 返回的 Meta 数据类类型，默认为 Meta
+
+    Returns:
+        更新后的 Meta 对象
+
+    Raises:
+        DataOpsError: API 返回错误时
+        AuthenticationError: 未认证或认证失败时
+    """
+    # body = dict(meta=meta)
+    body = dict(meta=meta.model_dump(mode="json", exclude_defaults=True))
+    # body = {
+    #     "meta": deep_merge(
+    #         origin_meta.model_dump(exclude_defaults=True), meta.model_dump(exclude_defaults=True)
+    #     )
+    # }
+    # old: 2076921409387933698
+    # new: 2076921409387933698
+    if datasource:
+        headers = headers or {}
+        headers["x-sub-system"] = datasource
+    # with open('update.json', 'wb') as f:
+    #     f.write(to_json(body, indent=2))
+
+    # try:
+    #     with get_session() as session:
+    #         resp = session.post(
+    #             "/meta/release/list/update",
+    #             headers=headers,
+    #             content=to_json(encrypt_body(body)),
+    #         )
+    #         resp_body = resp.json()
+    #         if resp.status_code == 401:
+    #             raise AuthenticationError("认证失败，请先登录")
+    #         if resp.status_code != 200:
+    #             raise DataOpsError(
+    #                 resp_body.get("msg", f"HTTP {resp.status_code}"), resp.status_code
+    #             )
+
+    # except SystemExit as e:
+    #     raise AuthenticationError("认证失败，请先登录") from e
+    # if resp_body.get("data") is None:
+    #     raise FailedToUpdateError(f"创建失败")
+
+    # raw_meta = resp_body["data"]
+    # if not isinstance(raw_meta, dict):
+    # raise DataOpsError(f"返回的数据格式不正确, 预期为字典, 实际为 {type(raw_meta)}")
+
+    # return Meta[_class].model_validate(raw_meta)

dataops_sdk/services/metrics.py

@@ -0,0 +1,42 @@
+"""指标服务."""
+
+from collections.abc import Mapping
+
+from pydantic import JsonValue
+
+from dataops_sdk.schemas import Meta, MetaRelease, MetaType
+from dataops_sdk.services.meta import release_list_get_by_code, release_list_list
+from dataops_sdk.utils import filter_nodes_by_pattern
+
+__all__ = ["service_list_metrics", "service_get_metrics"]
+
+
+def service_list_metrics(query: str | None = None) -> list[MetaRelease]:
+    """列出指标（可选正则过滤）.
+
+    Args:
+        query: 正则表达式，匹配 NAME / TITLE / REMARK / DIGEST
+
+    Returns:
+        MetaRelease 对象列表
+
+    Raises:
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证时
+    """
+    nodes = release_list_list(MetaType.METRICS)
+    if query:
+        nodes = filter_nodes_by_pattern(nodes, query, include_digest=True)
+    return nodes
+
+
+def service_get_metrics(code: str) -> Meta[Mapping[str, JsonValue]]:
+    """获取指标详情.
+
+    Args:
+        code: 指标编码
+
+    Returns:
+        指标元数据。`meta.data` 结构详见后端指标定义。
+    """
+    return release_list_get_by_code(MetaType.METRICS, code)

dataops_sdk/services/ref.py

@@ -0,0 +1,66 @@
+from collections.abc import Sequence
+
+from dataops_sdk.schemas import MetaRef, MetaType
+from dataops_sdk.services.meta import release_ref_list, release_ref_list_by_ref
+
+__all__ = ["service_ref_forward", "service_ref_backward"]
+
+
+def service_ref_forward(
+    meta_type: MetaType | str,
+    code: str,
+    ref_types: Sequence[str] | None = None,
+    *,
+    deduplicate: bool = False,
+) -> list[MetaRef]:
+    """正向引用查询：查询自己引用了谁。
+
+    Args:
+        meta_type: 元数据类型 (如 "table")
+        code: 元数据编码
+        ref_types: 仅返回指定的被引用类型过滤（如 ["table", "dict"]）
+        deduplicate: 是否去重
+
+    Returns:
+        MetaRef 对象列表
+    """
+    results = release_ref_list(meta_type, code)
+    if ref_types:
+        results = [r for r in results if r.refType in ref_types]
+    return deduplicate_refs(results) if deduplicate else results
+
+
+def service_ref_backward(
+    meta_type: MetaType | str,
+    code: str,
+    ref_types: Sequence[str] | None = None,
+    *,
+    deduplicate: bool = False,
+) -> list[MetaRef]:
+    """反向引用查询：查询谁引用了自己。
+
+    Args:
+        meta_type: 被引用类型 (如 "table")
+        code: 被引用编码
+        ref_types: 仅返回指定的引用类型过滤（如 ["table", "dict"]）
+        deduplicate: 是否去重
+
+    Returns:
+        MetaRef 对象列表
+    """
+    results = release_ref_list_by_ref(meta_type, code)
+    if ref_types:
+        results = [r for r in results if r.type in ref_types]
+    return deduplicate_refs(results) if deduplicate else results
+
+
+def deduplicate_refs(refs: list[MetaRef]) -> list[MetaRef]:
+    """根据 (code, type) 去重。"""
+    seen: set[tuple[str, str]] = set()
+    deduped: list[MetaRef] = []
+    for r in refs:
+        key = (r.code, r.type)
+        if key not in seen:
+            seen.add(key)
+            deduped.append(r)
+    return deduped

dataops_sdk/services/report.py

@@ -0,0 +1,36 @@
+from dataops_sdk.schemas import MetaRelease, MetaType, ReportMetaData
+from dataops_sdk.services import release_list_get_by_code, release_list_list
+from dataops_sdk.utils import filter_nodes_by_pattern
+
+__all__ = ["service_get_report", "service_list_reports"]
+
+
+def service_list_reports(query: str | None = None) -> list[MetaRelease]:
+    """列出文档（可选正则过滤）.
+
+    Args:
+        query: 正则表达式，匹配 NAME / TITLE / REMARK / DIGEST
+
+    Returns:
+        MetaRelease 对象列表
+
+    Raises:
+        DataOpsError: API 请求失败时
+        AuthenticationError: 未认证时
+    """
+    nodes = release_list_list(MetaType.REPORT)
+    if query:
+        nodes = filter_nodes_by_pattern(nodes, query, include_digest=True)
+    return nodes
+
+
+def service_get_report(code: str) -> ReportMetaData:
+    """读取文档内容并渲染为 Markdown.
+
+    Args:
+        code: 文档编码
+
+    Returns:
+        渲染后的 Markdown 字符串
+    """
+    return release_list_get_by_code(MetaType.REPORT, code, _class=ReportMetaData)