dts-dance 0.1.9__py3-none-any.whl → 0.2.1__py3-none-any.whl

{dts_dance-0.1.9.dist-info → dts_dance-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dts-dance
-Version: 0.1.9
+Version: 0.2.1
 Summary: dts dance lib
 Keywords: observation,tools
 Requires-Python: >=3.12

dts_dance-0.2.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+dtsdance/__init__.py,sha256=Yl_jEZ5weYfcrklnDvwB4wSgCOvMBLRRgWx0gHs3qfM,49
+dtsdance/bytecloud.py,sha256=SN82KIYAfq16s06-HUoj37PbgSbUa7MoFFcah2kaXOk,5213
+dtsdance/dflow.py,sha256=N9u7uB9UyUbB2FswBezXB9SbwmlDr3PWhY_eUZJeQ-8,7064
+dtsdance/dsyncer.py,sha256=X59sKDvK6rDeumCajepcqSSJysDfvJByYZAhcdOyNLw,11179
+dtsdance/feishu_base.py,sha256=2j4ZM4PFqJ-9EhC6DQ1OmAg--3VBGZyyRuxyjL0j6OU,3733
+dtsdance/feishu_table.py,sha256=ZUeoKrM4nmm5hFhc3vWOVYeLP390orm-284Od92G4iQ,8424
+dtsdance/metrics_fe.py,sha256=hzIl5BJmuCrkqJOHELVzXm3YAqrPttbyVkKBglS4mgQ,18978
+dtsdance/s3.py,sha256=Bh-cwLksfO5PewNtIzE_Md3rRLDLI1DUVoOD7Pou5T8,1294
+dtsdance/spacex.py,sha256=eYPx3-VKgPK6-IO8-HPuX0-mxsM6kt1HMZqyDA_KPbw,2340
+dtsdance/tcc_inner.py,sha256=xy3N1N3BNl1oc2oMNHKhnsj47rAKOOJ-aXw8GtA2De8,1621
+dtsdance/tcc_open.py,sha256=Qb_ue3xH0CSsoReItdFKuvW7JsDTfFkDNIB_4zjzSQI,6893
+dts_dance-0.2.1.dist-info/METADATA,sha256=QQohyobEcAlWjdpCDwPBXzYlSRBZxeGcV-xfN9_ladU,793
+dts_dance-0.2.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dts_dance-0.2.1.dist-info/RECORD,,

dtsdance/bytecloud.py

@@ -6,27 +6,29 @@ from loguru import logger
 from typing import NamedTuple
 
 
-class ByteCloudEnvInfo(NamedTuple):
+class SiteConfig(NamedTuple):
     name: str
     endpoint: str
-    token: str
+    svc_account: str
+    svc_secret: str
+    endpoint_bytedts_spacex: str | None = None
 
 
-class ByteCloudHelper:
+class ByteCloudClient:
     """
-    ByteCloud Helper
+    ByteCloud Client
     """
 
     # 每小时刷新一次，单位为秒
     _REFRESH_INTERVAL = 1 * 60 * 60
 
-    def __init__(self, envs: dict[str, ByteCloudEnvInfo]):
+    def __init__(self, sites: dict[str, SiteConfig]):
         """
-        初始化 ByteCloud Helper
+        初始化 ByteCloud Client
         从配置文件加载所有环境的信息，并为每个环境初始化 JWT 令牌
-        envs 中保存内容 list[(name, endpoint, token)]
+        sites 中保存内容 list[(name, endpoint, svc_account, svc_secret)]
         """
-        self.envs = envs
+        self.sites = sites
 
         # 初始化线程锁，用于保护 jwt_tokens 的并发访问
         self.token_lock = threading.Lock()
@@ -66,20 +68,20 @@ class ByteCloudHelper:
         """
         logger.debug("开始刷新所有环境的 JWT 令牌...")
 
-        for _, env in self.envs.items():
+        for _, site in self.sites.items():
             try:
                 # 刷新令牌
-                new_token = self._acquire_jwt_token(env.endpoint, env.token)
+                new_token = self._acquire_jwt_token(site.endpoint, site.svc_secret)
                 # 使用线程锁更新缓存中的 JWT 令牌
                 with self.token_lock:
-                    self.jwt_tokens[env.name] = new_token
-                    logger.debug(f"环境 {env.name} 的 JWT 令牌成功刷新，新令牌: {new_token}")
+                    self.jwt_tokens[site.name] = new_token
+                    logger.debug(f"环境 {site.name} 的 JWT 令牌成功刷新，新令牌: {new_token}")
             except Exception as e:
-                logger.error(f"环境 {env.name} 的 JWT 令牌刷新失败: {e}")
+                logger.error(f"环境 {site.name} 的 JWT 令牌刷新失败: {e}")
 
         logger.debug(f"所有环境的 JWT 令牌已成功刷新。jwt_tokens: {self.jwt_tokens}")
 
-    def _acquire_jwt_token(self, endpoint: str, token: str) -> str:
+    def _acquire_jwt_token(self, endpoint: str, svc_secret: str) -> str:
         """
         获取 JWT 令牌
 
@@ -93,7 +95,7 @@ class ByteCloudHelper:
         url = endpoint + "/auth/api/v1/jwt"
         headers = {
             "Content-Type": "application/json",
-            "Authorization": "Bearer " + token,
+            "Authorization": "Bearer " + svc_secret,
         }
 
         try:
@@ -125,39 +127,43 @@ class ByteCloudHelper:
             logger.error(f"获取JWT令牌时出错: {e}")
             raise
 
-    def get_jwt_token(self, env: str) -> str:
+    def get_jwt_token(self, site: str) -> str:
         """
-        获取指定环境的 JWT 令牌
+        获取指定站点的 JWT 令牌
+        """
+        # 使用线程锁保护并发访问
+        with self.token_lock:
+            if site not in self.jwt_tokens:
+                raise KeyError(f"站点 {site} 的 JWT 令牌不存在")
+            return self.jwt_tokens[site]
+
+    def build_request_headers(self, site: str) -> dict[str, str]:
+        """
+        构建请求头
 
         Args:
-            env: 环境名称，如 'China-North', 'China-East', 'Singapore-Central' 等
+            site: 站点名称
 
         Returns:
-            str: 当前有效的 JWT 令牌
-
-        Raises:
-            KeyError: 如果指定的环境不存在
+            dict[str, str]: 请求头字典
         """
-        # 使用线程锁保护并发访问
-        with self.token_lock:
-            if env not in self.jwt_tokens:
-                raise KeyError(f"环境 {env} 的 JWT 令牌不存在")
-            return self.jwt_tokens[env]
+        jwt_token = self.get_jwt_token(site)
+        return {"Content-Type": "application/json", "x-jwt-token": jwt_token}
 
-    def get_env_info(self, env: str) -> ByteCloudEnvInfo:
+    def get_site_info(self, site: str) -> SiteConfig:
         """
         获取指定环境的信息
 
         Args:
-            env: 环境名称
+            site: 站点名称
 
         Returns:
-            ByteCloudEnvInfo: 指定环境的信息
+            SiteConfig: 指定站点的信息
 
         Raises:
-            KeyError: 如果指定的环境不存在
+            KeyError: 如果指定的站点不存在
         """
-        if env not in self.envs:
-            raise KeyError(f"环境 {env} 不存在")
+        if site not in self.sites:
+            raise KeyError(f"站点 {site} 不存在")
 
-        return self.envs[env]
+        return self.sites[site]

dtsdance/dflow.py

@@ -1,27 +1,21 @@
 from typing import Any, cast, Optional
 from loguru import logger
-from .bytecloud import ByteCloudHelper
+from .bytecloud import ByteCloudClient
 import requests
 
 
-class DFlowHelper:
+class TaskNotFound(Exception):
+    pass
 
-    def __init__(self, bytecloud_helper: ByteCloudHelper) -> None:
-        self.bytecloud_helper = bytecloud_helper
 
-    def _build_headers(self, env: str) -> dict[str, str]:
-        """
-        构建请求头
+class DFlowNotFound(Exception):
+    pass
 
-        Args:
-            env: 环境名称
 
-        Returns:
-            dict[str, str]: 请求头字典
-        """
-        jwt_token = self.bytecloud_helper.get_jwt_token(env)
-        headers = {"x-jwt-token": jwt_token}
-        return headers
+class DFlowClient:
+
+    def __init__(self, bytecloud_client: ByteCloudClient) -> None:
+        self.bytecloud_client = bytecloud_client
 
     def _make_request(self, method: str, url: str, headers: dict[str, str], json_data: Optional[dict] = None) -> dict[str, Any]:
         """
@@ -58,30 +52,31 @@ class DFlowHelper:
             logger.warning(error_msg)
             raise
 
-    def get_dflow_info(self, env: str, task_id: str) -> dict[str, Any]:
+    def get_task_info(self, site: str, task_id: str) -> dict[str, Any]:
         """
         获取 DFlow 任务信息
 
         Args:
-            env: 环境名称
+            site: 站点名称
             task_id: DFlow 任务 ID
 
         Returns:
             dict[str, Any]: DFlow 任务信息，包含 create_time 等字段
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeTaskInfo"
-
-        # 准备请求头
-        headers = self._build_headers(env)
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeTaskInfo"
 
         # 构建请求数据
         json_data = {"id": int(task_id)}
 
-        response_data = self._make_request("POST", url, headers, json_data)
+        response_data = self._make_request("POST", url, self.bytecloud_client.build_request_headers(site), json_data)
+
+        message = response_data.get("message")
+        # logger.debug(f"get_task_info {site} {task_id}, message: {message}")
 
-        logger.info(f"get_dflow_info {env} {task_id}, message: {response_data.get('message')}")
+        if message == "task not exists":
+            raise TaskNotFound(f"获取 DFlow 任务信息失败，站点: {site}, 任务 ID: {task_id} 不存在")
 
         try:
             data = cast(dict, response_data.get("data", {}))
@@ -99,71 +94,108 @@ class DFlowHelper:
         except (KeyError, AttributeError, Exception) as e:
             raise Exception(f"无法从响应中提取 DFlow 任务信息数据: {str(e)}")
 
-    def generate_task_url(self, env: str, task_id: str) -> str:
+    def get_dflow_info(self, site: str, dflow_id: str) -> dict[str, Any]:
+        """
+        获取 DFlow 进程信息
+
+        Args:
+            site: 站点名称
+            dflow_id: DFlow 进程 ID
+
+        Returns:
+            dict[str, Any]: DFlow 进程信息，包含 create_time 等字段
+        """
+        # 构建 API URL
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeDFlowDetail"
+
+        # 构建请求数据
+        json_data = {"dflow_id": int(dflow_id)}
+
+        response_data = self._make_request("POST", url, self.bytecloud_client.build_request_headers(site), json_data)
+
+        message = response_data.get("message", "")
+        # logger.debug(f"get_dflow_info {site} {dflow_id}, message: {message}")
+
+        if "dflow not found" in message:
+            raise DFlowNotFound(f"获取 DFlow 进程信息失败，站点: {site}, 进程 ID: {dflow_id} 不存在")
+
+        try:
+            data = cast(dict, response_data.get("data", {}))
+            dflow = cast(dict, data.get("dflow", {}))
+            # 提取核心信息
+            filtered_data = {
+                "dflow_id": dflow.get("id", ""),
+                "task_id": dflow.get("task_id", ""),
+                "app": dflow.get("app", ""),
+                "schedule_plan_name": dflow.get("schedule_plan_name", ""),
+                "running_state.healthy_status": dflow.get("running_state", {}).get("healthy_status", ""),
+            }
+
+            return filtered_data
+
+        except (KeyError, AttributeError, Exception) as e:
+            raise Exception(f"无法从响应中提取 DFlow 进程信息数据: {str(e)}")
+
+    def generate_task_url(self, site: str, task_id: str) -> str:
         """
         获取 DFlow 任务详情页面的 URL
 
         Args:
-            env: 环境名称
+            site: 站点名称
             task_id: DFlow 任务 ID
 
         Returns:
             str: DFlow 任务详情页面的 URL
         """
         # 根据环境生成对应的 scope 参数
-        env_info = self.bytecloud_helper.get_env_info(env)
-        return f"{env_info.endpoint}/bytedts/datasync/detail/{task_id}?scope={env}"
+        site_info = self.bytecloud_client.get_site_info(site)
+        return f"{site_info.endpoint}/bytedts/datasync/detail/{task_id}?scope={site}"
 
-    def init_resources(self, env: str, ctrl_env: str) -> bool:
+    def init_resources(self, site: str, ctrl_env: str) -> bool:
         """
         初始化 CTRL 环境资源
 
         Args:
-            env: 环境名称
+            site: 站点名称
             ctrl_env: 控制环境
 
         Returns:
             bool: CTRL 环境资源初始化结果
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/bytedts/api/bytedts/v3/InitSystemResource"
-
-        # 准备请求头
-        headers = self._build_headers(env)
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/InitSystemResource"
 
         # 构建请求数据
         json_data = {"ctrl_env": ctrl_env}
 
-        response_data = self._make_request("POST", url, headers, json_data)
+        response_data = self._make_request("POST", url, self.bytecloud_client.build_request_headers(site), json_data)
 
         message = response_data.get("message")
-        logger.info(f"int_resources {env} {ctrl_env}, message: {message}")
+        logger.info(f"int_resources {site} {ctrl_env}, message: {message}")
 
         return message == "ok"
 
-    def list_resources(self, env: str, ctrl_env: str) -> list[str]:
+    def list_resources(self, site: str, ctrl_env: str) -> list[str]:
         """
         列举 CTRL 环境资源列表
 
         Args:
-            env: 环境名称
+            site: 站点名称
             ctrl_env: 控制环境
 
         Returns:
             list[str]: CTRL 环境资源列表
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeResources"
-
-        # 准备请求头
-        headers = self._build_headers(env)
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeResources"
 
         # 构建请求数据
         json_data = {"offset": 0, "limit": 10, "ctrl_env": ctrl_env}
 
-        response_data = self._make_request("POST", url, headers, json_data)
+        response_data = self._make_request("POST", url, self.bytecloud_client.build_request_headers(site), json_data)
 
         try:
             data = cast(dict, response_data.get("data", {}))

dtsdance/dsyncer.py

@@ -1,4 +1,4 @@
-from .bytecloud import ByteCloudHelper
+from .bytecloud import ByteCloudClient
 from typing import Any, NamedTuple, Tuple, cast, Optional
 from loguru import logger
 import requests
@@ -15,11 +15,11 @@ class DSyncerEnvInfo(NamedTuple):
     token: str
 
 
-class DSyncerHelper:
+class DSyncerClient:
 
-    def __init__(self, envs: dict[str, DSyncerEnvInfo], bytecloud_helper: ByteCloudHelper) -> None:
+    def __init__(self, envs: dict[str, DSyncerEnvInfo], bytecloud_client: ByteCloudClient) -> None:
         self.envs = envs
-        self.bytecloud_helper = bytecloud_helper
+        self.bytecloud_client = bytecloud_client
 
     def _build_headers(self, env: str, secret_api: bool = False) -> dict[str, str]:
         """
@@ -32,7 +32,7 @@ class DSyncerHelper:
         Returns:
             dict[str, str]: 请求头字典
         """
-        jwt_token = self.bytecloud_helper.get_jwt_token(env)
+        jwt_token = self.bytecloud_client.get_jwt_token(env)
         headers = {"X-Jwt-Token": jwt_token, "x-bcgw-vregion": env}
 
         if secret_api:
@@ -41,7 +41,7 @@ class DSyncerHelper:
 
         return headers
 
-    def _make_request(self, method: str, url: str, headers: dict[str, str], json_data: Optional[dict] = None) -> dict[str, Any]:
+    def _make_request(self, method: str, url: str, headers: dict[str, str], data_raw: Optional[dict] = None) -> dict[str, Any]:
         """
         发送 HTTP 请求的通用方法
 
@@ -60,7 +60,7 @@ class DSyncerHelper:
             if method.upper() == "GET":
                 response = requests.get(url, headers=headers)
             elif method.upper() == "POST":
-                response = requests.post(url, json=json_data, headers=headers)
+                response = requests.post(url, json=data_raw, headers=headers)
             else:
                 raise ValueError(f"不支持的 HTTP 方法: {method}")
 
@@ -77,41 +77,41 @@ class DSyncerHelper:
             logger.warning(error_msg)
             raise
 
-    def _acquire_task_info(self, env: str, task_id: str) -> dict[str, str]:
+    def _acquire_task_info(self, site: str, task_id: str) -> dict[str, str]:
         """
         获取 DSyncer 任务信息
 
         Args:
-            env: 环境名称
+            site: 站点名称
             task_id: DSyncer 任务 ID
 
         Returns:
             dict[str, str]: DSyncer 任务的 rocket_mq_connection 信息，只包含 cluster、topic 和 group 字段
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/dsyncer/openapi/taskinfo/{task_id}/"
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/dsyncer/openapi/taskinfo/{task_id}/"
 
         # 准备请求头
-        headers = self._build_headers(env)
+        headers = self._build_headers(site)
 
         return self._make_request("GET", url, headers)
 
-    def get_dflow_task_info(self, env: str, task_id: str) -> tuple[str, str]:
+    def get_dflow_task_info(self, site: str, task_id: str) -> tuple[str, str]:
         """
         获取迁移后的 DFlow 任务信息
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/dsyncer/openapi/taskinfo/{task_id}/migrate/"
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/dsyncer/openapi/taskinfo/{task_id}/migrate/"
 
         # 准备请求头
-        headers = self._build_headers(env)
+        headers = self._build_headers(site)
 
         json_data = self._make_request("GET", url, headers)
 
         message = json_data.get("message", "")
-        logger.debug(f"get task migrate info {env} {task_id}, message: {message}")
+        logger.debug(f"get task migrate info {site} {task_id}, message: {message}")
 
         # 从消息中提取 DFlow 任务 URL 和 ID
         # 消息格式: "任务已迁移至ByteDTS平台，请前往[https://cloud.bytedance.net/bytedts/datasync/detail/93127366537986?scope=China-North&tabKey=DetailInfo]查看详情"
@@ -133,7 +133,7 @@ class DSyncerHelper:
             logger.warning(f"could not extract dflow task info from message: {message}")
             return ("", "")
 
-    def generate_task_url(self, env: str, task_id: str) -> str:
+    def generate_task_url(self, site: str, task_id: str) -> str:
         """
         获取 DSyncer 任务详情页面的 URL
 
@@ -144,8 +144,8 @@ class DSyncerHelper:
         Returns:
             str: DSyncer 任务详情页面的 URL
         """
-        env_info = self.bytecloud_helper.get_env_info(env)
-        return DSyncer_Task_Detail_URL.format(endpoint=env_info.endpoint, task_id=task_id)
+        site_info = self.bytecloud_client.get_site_info(site)
+        return DSyncer_Task_Detail_URL.format(endpoint=site_info.endpoint, task_id=task_id)
 
     def generate_task_grafana_url(self, task_id: str, change_time: str) -> str:
         """
@@ -200,7 +200,7 @@ class DSyncerHelper:
         except (KeyError, AttributeError, Exception) as e:
             raise Exception(f"无法从响应中提取 DSyncer 任务状态数据: {str(e)}")
 
-    def is_task_migrate_running(self, env: str, task_id: str) -> Tuple[bool, str]:
+    def is_task_migrate_running(self, site: str, task_id: str) -> Tuple[bool, str]:
         """
         检查 DSyncer 任务是否正在迁移中
 
@@ -208,62 +208,62 @@ class DSyncerHelper:
             bool: 如果任务正在迁移中，返回 True；否则返回 False
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/dsyncer/secret_api/task/migrate/check"
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/dsyncer/secret_api/task/migrate/check"
 
         # 准备请求头
-        headers = self._build_headers(env, secret_api=True)
+        headers = self._build_headers(site, secret_api=True)
 
-        json_data = {"task_id": task_id}
+        data_raw = {"task_id": task_id}
 
-        response_data = self._make_request("POST", url, headers, json_data)
+        response_data = self._make_request("POST", url, headers, data_raw)
 
         message = response_data.get("message", "")
-        logger.debug(f"get task migrate status {env} {task_id}, message: {message}")
+        logger.debug(f"get task migrate status {site} {task_id}, message: {message}")
         if "task migrate is running" in message:
             return True, ""
         else:
             return False, response_data.get("data", {}).get("msg", {})
 
-    def migration_rollback(self, env: str, task_id: str) -> bool:
+    def migration_rollback(self, site: str, task_id: str) -> bool:
         """
         执行回滚
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/dsyncer/secret_api/task/rollback_migrate2dsyncer/"
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/dsyncer/secret_api/task/rollback_migrate2dsyncer/"
 
         # 准备请求头
-        headers = self._build_headers(env, secret_api=True)
+        headers = self._build_headers(site, secret_api=True)
 
-        json_data = {"task_id_list": [task_id]}
+        data_raw = {"task_id_list": [task_id]}
 
-        response_data = self._make_request("POST", url, headers, json_data)
+        response_data = self._make_request("POST", url, headers, data_raw)
 
-        logger.debug(f"migration_mark_rollback return {env} {task_id}, json_data: {response_data}")
+        logger.debug(f"migration_mark_rollback return {site} {task_id}, json_data: {response_data}")
         success_task = response_data.get("data", {}).get("success_task", [])
         return task_id in success_task
 
-    def migration_mark_success(self, env: str, task_id: str) -> bool:
+    def migration_mark_success(self, site: str, task_id: str) -> bool:
         """
         标记迁移成功
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/dsyncer/secret_api/task/mark_migrate_success/"
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/dsyncer/secret_api/task/mark_migrate_success/"
 
         # 准备请求头
-        headers = self._build_headers(env, secret_api=True)
+        headers = self._build_headers(site, secret_api=True)
 
-        json_data = {"task_id": task_id}
+        data_raw = {"task_id": task_id}
 
-        response_data = self._make_request("POST", url, headers, json_data)
+        response_data = self._make_request("POST", url, headers, data_raw)
 
-        logger.debug(f"migration_mark_success return {env} {task_id}, json_data: {response_data}")
+        logger.debug(f"migration_mark_success return {site} {task_id}, json_data: {response_data}")
         success_task = response_data.get("data", {}).get("success_task", [])
         return task_id in success_task
 
-    def migrate_task(self, env: str, task_id: str, app_parallel: int) -> Optional[str]:
+    def migrate_task(self, site: str, task_id: str, app_parallel: int) -> Optional[str]:
         """
         迁移任务到DFlow
 
@@ -271,13 +271,13 @@ class DSyncerHelper:
             Optional[str]: 错误信息，成功时返回None，失败时返回错误信息
         """
         # 构建 API URL
-        env_info = self.bytecloud_helper.get_env_info(env)
-        url = f"{env_info.endpoint}/api/v1/dsyncer/secret_api/task/migrate2dflow/single"
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v1/dsyncer/secret_api/task/migrate2dflow/single"
 
         # 准备请求头
-        headers = self._build_headers(env, secret_api=True)
+        headers = self._build_headers(site, secret_api=True)
 
-        json_data = {
+        data_raw = {
             "task_id": task_id,
             "delay_threshold": "20s",
             "dsyncer_task_pause_threshold": "180s",
@@ -289,9 +289,9 @@ class DSyncerHelper:
             "app_parallel": app_parallel,
         }
 
-        response_data = self._make_request("POST", url, headers, json_data)
+        response_data = self._make_request("POST", url, headers, data_raw)
 
-        logger.debug(f"migrate_task return {env} {task_id}, json_data: {response_data}")
+        logger.debug(f"migrate_task return {site} {task_id}, json_data: {response_data}")
         message = response_data.get("message")
         if response_data.get("code") == 0 and message == "ok":
             err_message = None

dtsdance/metrics_fe.py

@@ -2,7 +2,7 @@ from loguru import logger
 import requests
 import threading
 import time
-from typing import Dict, List, NamedTuple, Optional
+from typing import NamedTuple, Optional
 from datetime import datetime, timedelta
 
 
@@ -60,9 +60,9 @@ def get_metric_type_by_name(metric_name: str) -> Optional[MetricType]:
     return None
 
 
-class MetricsHelper:
+class MetricsClient:
     """
-    Metrics Helper
+    Metrics Client
     """
 
     # https://cloud.bytedance.net/docs/metrics/docs/63bbbb1ec6b537022a6000c7/63bbd4e8777a300220d4ac3a?x-resource-account=public&x-bc-region-id=bytedance
@@ -71,7 +71,7 @@ class MetricsHelper:
 
     def __init__(self, envs: dict[str, MetricEnvInfo]):
         """
-        初始化 Metrics Helper
+        初始化 Metrics Client
         从配置文件加载所有环境的信息，并为每个环境初始化访问令牌
         """
         self.envs = envs
@@ -80,7 +80,7 @@ class MetricsHelper:
         self.token_lock = threading.Lock()
 
         # 初始化访问令牌缓存，按环境名称索引
-        self.tokens: Dict[str, str] = {}
+        self.tokens: dict[str, str] = {}
 
         # 更新所有环境的配置信息
         self._refresh_tokens()
@@ -212,8 +212,8 @@ class MetricsHelper:
         return self.envs[env_name].endpoint
 
     def _build_query_payload(
-        self, metric_name: str, aggregator: str, start_time_ms: int, end_time_ms: int, filters: List[Dict], rate: bool, **kwargs
-    ) -> Dict:
+        self, metric_name: str, aggregator: str, start_time_ms: int, end_time_ms: int, filters: list[dict], rate: bool, **kwargs
+    ) -> dict:
         """
         构建查询请求体
 
@@ -254,8 +254,8 @@ class MetricsHelper:
         }
 
     def _execute_metrics_query(
-        self, env: str, metric_name: str, aggregator: str, start_time: str, end_time: str, filters: List[Dict], rate: bool, **kwargs
-    ) -> Dict:
+        self, env: str, metric_name: str, aggregator: str, start_time: str, end_time: str, filters: list[dict], rate: bool, **kwargs
+    ) -> dict:
         """
         执行通用的指标查询
 
@@ -318,7 +318,7 @@ class MetricsHelper:
 
     def get_mq_group_metrics(
         self, env: str, cluster: str, topic: str, group: str, start_time: str, end_time: str, metric_type: MetricType, **kwargs
-    ) -> Dict:
+    ) -> dict:
         """
         获取指定时间段的堆积指标
 
@@ -398,7 +398,7 @@ class MetricsHelper:
 
         return int(max_consume_tps)
 
-    def get_dsyncer_task_metrics(self, env: str, metric_type: MetricType, dsyncer_increment_task_id: str, start_time: str, end_time: str) -> Dict:
+    def get_dsyncer_task_metrics(self, env: str, metric_type: MetricType, dsyncer_increment_task_id: str, start_time: str, end_time: str) -> dict:
         """
         获取 DSyncer 任务指定时间段的指标
 
@@ -434,7 +434,7 @@ class MetricsHelper:
             logger.error(f"get_task_latency_metrics env: {env}, dsyncer_increment_task_id: {dsyncer_increment_task_id}, error: {e}")
             raise Exception(f"获取指标失败: {e}")
 
-    def get_dflow_task_metrics(self, env: str, dflow_task_id: str, metric_type: MetricType, start_time: str, end_time: str, **kwargs) -> Dict:
+    def get_dflow_task_metrics(self, env: str, dflow_task_id: str, metric_type: MetricType, start_time: str, end_time: str, **kwargs) -> dict:
         """
         获取指定时间段的 DFlow 指标
 
@@ -471,7 +471,7 @@ class MetricsHelper:
             raise Exception(f"获取指标失败: {e}")
 
     @staticmethod
-    def _remove_last_datapoint(end_time: str, dps: Dict[str, float]) -> Dict[str, float]:
+    def _remove_last_datapoint(end_time: str, dps: dict[str, float]) -> dict[str, float]:
         """
         移除监控数据中最后一个数据点，因为最后一个点往往不准确
         只有当最后一个点的时间戳与 end_time 相差在 15 秒内时才移除

dtsdance/spacex.py

@@ -1,69 +1,68 @@
-from .bytecloud import ByteCloudHelper
+from typing import Any, NamedTuple
+from .bytecloud import ByteCloudClient
 import requests
-import json
-from typing import Dict, List
 from loguru import logger
 
 
-class SpaceXNotifier:
+class GatewayInfo(NamedTuple):
+    mgr_env: str
+    ctrl_name: str
+    gateway_endpoint: str
+    auth_user: str
+    auth_password: str
+    root_secret_key: str
+    gw_meta_db: str
+
+
+class SpaceXClient:
     """
     SpaceX 通知服务客户端，用于发送飞书消息
     """
 
-    # 飞书机器人 webhook URL
-    _SPACEX_DOMAIN = {
-        "BOE": "spacex-api-boe.byted.org",
-        "China-North": "spacex-api.byted.org",
-        "Singapore-Central": "spacex-api-i18n.byted.org",
-        "EUTTP": "spacex-api.tiktoke.org",
-    }
-
-    def __init__(self, bytecloud_helper: ByteCloudHelper):
+    def __init__(self, bytecloud_client: ByteCloudClient):
         """
         初始化 SpaceX 通知客户端
         """
-        self.bytecloud_helper = bytecloud_helper
+        self.bytecloud_client = bytecloud_client
 
-    def send_feishu_message(self, env: str, feishu_content: Dict, feishu_groups: List[str]) -> str:
-        """
-        发送飞书消息
+    def list_mgr(self, site: str) -> list[Any]:
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint_bytedts_spacex}/bytedts/v1/queryServerMeta"
 
-        Args:
-            feishu_content: 飞书消息内容，包含卡片配置
-            feishu_groups: 飞书群组 ID 列表
+        try:
+            response = requests.post(url, headers=self.bytecloud_client.build_request_headers(site))
+            response.raise_for_status()
 
-        Returns:
-            str: 响应内容
+            result = response.json()
+            return result.get("data", [])
 
-        Raises:
-            Exception: 请求失败时抛出异常
-        """
-        logger.debug(f"send_feishu_message env: {env}, feishu_content: {str(feishu_content)[:100]}..., feishu_groups: {feishu_groups}")
+        except Exception as e:
+            logger.warning(f"do quest queryServerMeta exception: {e}")
+            raise
 
-        body = {
-            "product_name": "ByteDTS-Inspector",
-            "notification_detail": [
-                {
-                    "channel": "Lark",
-                    "content": json.dumps(feishu_content),
-                    "lark_groups": feishu_groups,
-                }
-            ],
+    def register_gateway(self, site: str, gateway_info: GatewayInfo) -> bool:
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint_bytedts_spacex}/bytedts/v1/registryGateway"
+        data_raw = {
+            "server_region": gateway_info.mgr_env,
+            "cluster_region": gateway_info.ctrl_name,
+            "cluster_name": gateway_info.ctrl_name,
+            "server_domain": gateway_info.gateway_endpoint,
+            "frontend_user": gateway_info.auth_user,
+            "gateway_user": gateway_info.auth_user,
+            "root_secret_key": gateway_info.root_secret_key,
+            "gateway_password": gateway_info.auth_password,
+            "frontend_password": gateway_info.auth_password,
+            "gw_meta_db": gateway_info.gw_meta_db,
+            "gateway_type": "psm",
         }
-
-        # 发送 HTTP POST 请求
         try:
-            url = f"https://{self._SPACEX_DOMAIN[env]}/notification_service/api/v1/messages"
-            headers = {
-                "Content-Type": "application/json",
-                "X-Jwt-Token": self.bytecloud_helper.get_jwt_token(env),
-            }
-
-            response = requests.post(url, json=body, headers=headers)
+            response = requests.post(url, json=data_raw, headers=self.bytecloud_client.build_request_headers(site))
             response.raise_for_status()
 
-            print("response: {}".format(response.text))
-            return response.text
+            result = response.json()
+            return result.get("message", "") == "ok"
+
         except Exception as e:
-            print("发送消息时出错: {}".format(e))
+            logger.warning(f"do quest registryGateway exception: {e}")
             raise

dtsdance/tcc_inner.py

@@ -0,0 +1,55 @@
+"""TCC (Toutiao Config Center) API client."""
+
+from typing import Any
+import requests
+
+from dtsdance.bytecloud import ByteCloudClient
+from dtsdance.tcc_open import TCCError
+
+
+class TCCInnerClient:
+    """Client for TCC OpenAPI."""
+
+    def __init__(self, bytecloud_client: ByteCloudClient) -> None:
+        self.bytecloud_client = bytecloud_client
+
+    def list_configs(
+        self,
+        site: str,
+        ns_name: str,
+        region: str,
+        dir: str,
+        conf_name: str,
+    ) -> dict[str, Any]:
+        """
+        List TCC configurations.
+        """
+        site_info = self.bytecloud_client.get_site_info(site)
+        url = f"{site_info.endpoint}/api/v3/tcc/bcc/config/list_v2"
+
+        data_raw = {
+            "ns_name": ns_name,
+            "region": region,
+            "dir_path": dir,
+            "keyword": conf_name,
+            "env": "prod",
+            "scope": "all",
+            "condition": "name",
+            "pn": 1,
+            "rn": 100,
+        }
+
+        try:
+            response = requests.post(url, headers=self.bytecloud_client.build_request_headers(site), json=data_raw, timeout=30)
+            response.raise_for_status()
+            result = response.json()
+
+            # Check for errors in response
+            base_resp = result.get("base_resp", {})
+            if base_resp.get("error_code", 0) != 0:
+                raise TCCError(f"TCC API error: {base_resp.get('error_message', 'Unknown error')}")
+
+            return result.get("data", {})
+
+        except requests.RequestException as e:
+            raise TCCError(f"Failed to get TCC config: {str(e)}") from e

dtsdance/{tcc.py → tcc_open.py}

@@ -1,4 +1,4 @@
-"""TCE (Toutiao Cloud Engine) API client."""
+"""TCC (Toutiao Config Center) API client."""
 
 from dataclasses import dataclass
 from typing import Any
@@ -16,19 +16,24 @@ class TCCConfigItem:
 
 
 class TCCError(Exception):
-    """TCC API error."""
+    pass
+
 
+class TCCItemNotFound(Exception):
     pass
 
 
 class TCCClient:
     """Client for TCC OpenAPI."""
 
-    def __init__(self, svc_account: str, svc_secret: str, base_url: str):
+    def __init__(self, svc_account: str, svc_secret: str, endpoint: str):
         self.svc_account = svc_account
-        self.base_url = base_url
-        self.headers = {
-            "Authorization": f"Bearer {svc_secret}",
+        self.svc_secret = svc_secret
+        self.endpoint = endpoint
+
+    def _build_headers(self) -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.svc_secret}",
         }
 
     def publish_config(
@@ -50,7 +55,7 @@ class TCCClient:
         Raises:
             TCCError: If API call fails
         """
-        url = f"{self.base_url}/api/v1/tcc_v3_openapi/bcc/open/config/update"
+        url = f"{self.endpoint}/api/v1/tcc_v3_openapi/bcc/open/config/update"
 
         payload = {
             "ns_name": ns_name,
@@ -67,7 +72,7 @@ class TCCClient:
 
         try:
             # print(f"writing tcc config, payload: {payload}")
-            response = requests.post(url, headers=self.headers, json=payload, timeout=30)
+            response = requests.post(url, headers=self._build_headers(), json=payload, timeout=30)
             if response.status_code != 200:
                 raise TCCError(f"TCC API request failed with status {response.status_code}\n" f"Response: {response.text}")
 
@@ -108,7 +113,7 @@ class TCCClient:
         Raises:
             TCCError: If API call fails
         """
-        url = f"{self.base_url}/api/v1/tcc_v3_openapi/bcc/open/config/list"
+        url = f"{self.endpoint}/api/v1/tcc_v3_openapi/bcc/open/config/list"
 
         params = {
             "ns_name": ns_name,
@@ -120,7 +125,7 @@ class TCCClient:
         }
 
         try:
-            response = requests.post(url, headers=self.headers, params=params, timeout=30)
+            response = requests.post(url, headers=self._build_headers(), params=params, timeout=30)
             response.raise_for_status()
             result = response.json()
 
@@ -191,8 +196,8 @@ class TCCClient:
         self,
         ns_name: str,
         region: str,
-        conf_name: str,
         dir: str,
+        conf_name: str,
     ) -> dict[str, Any]:
         """
         Get TCC configuration.
@@ -209,7 +214,7 @@ class TCCClient:
         Raises:
             TCCError: If API call fails
         """
-        url = f"{self.base_url}/api/v1/tcc_v3_openapi/bcc/open/config/get"
+        url = f"{self.endpoint}/api/v1/tcc_v3_openapi/bcc/open/config/get"
 
         params = {
             "ns_name": ns_name,
@@ -219,14 +224,18 @@ class TCCClient:
         }
 
         try:
-            response = requests.get(url, headers=self.headers, params=params, timeout=30)
+            response = requests.get(url, headers=self._build_headers(), params=params, timeout=30)
             response.raise_for_status()
             result = response.json()
 
             # Check for errors in response
             base_resp = result.get("base_resp", {})
             if base_resp.get("error_code", 0) != 0:
-                raise TCCError(f"TCC API error: {base_resp.get('error_message', 'Unknown error')}")
+                error_message = base_resp.get("error_message", "Unknown error")
+                if "reason:RESOURCE_NOT_FOUND" in error_message:
+                    raise TCCItemNotFound(f"TCC Item not found error: {error_message}")
+
+                raise TCCError(f"TCC API error: {error_message}")
 
             return result.get("data", {})
 

dts_dance-0.1.9.dist-info/RECORD

@@ -1,13 +0,0 @@
-dtsdance/__init__.py,sha256=Yl_jEZ5weYfcrklnDvwB4wSgCOvMBLRRgWx0gHs3qfM,49
-dtsdance/bytecloud.py,sha256=joixJEQxX9k3VqFDNrccTVSI2WVdxdZ1JyuWFVF9zjo,4998
-dtsdance/dflow.py,sha256=gow-zCX5lbQqqW2MRWvWynej7pZ_TQ5NmOb__j2aQT4,5497
-dtsdance/dsyncer.py,sha256=1OateZLGekYP_EF9_oAHqsj4sdQm6ahIoRllEgT9Y3I,11142
-dtsdance/feishu_base.py,sha256=2j4ZM4PFqJ-9EhC6DQ1OmAg--3VBGZyyRuxyjL0j6OU,3733
-dtsdance/feishu_table.py,sha256=ZUeoKrM4nmm5hFhc3vWOVYeLP390orm-284Od92G4iQ,8424
-dtsdance/metrics_fe.py,sha256=uPdbjGaaYBOv-rK7lKUx6aVT5Sj6ZkW9m06NyqMrPf0,18990
-dtsdance/s3.py,sha256=Bh-cwLksfO5PewNtIzE_Md3rRLDLI1DUVoOD7Pou5T8,1294
-dtsdance/spacex.py,sha256=wgbuwDTLXopJnLn2puX-9MfeVXvi_nQ3C9uN8McoJf4,2113
-dtsdance/tcc.py,sha256=M_0cOVYyvUgjnC1uKWdz3YgRplafQUqX39gX_YEg8eE,6563
-dts_dance-0.1.9.dist-info/METADATA,sha256=IbvejpKLh-EM12cf9XtlwCVkUbD5jjvRc_Pz7PA_5b8,793
-dts_dance-0.1.9.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dts_dance-0.1.9.dist-info/RECORD,,