dts-dance 0.2.4__py3-none-any.whl → 0.2.5__py3-none-any.whl

{dts_dance-0.2.4.dist-info → dts_dance-0.2.5.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: dts-dance
-Version: 0.2.4
+Version: 0.2.5
 Summary: dts dance lib
 Keywords: observation,tools
 Requires-Python: >=3.12
-Requires-Dist: boto3<2.0.0,>=1.42.30
+Requires-Dist: boto3>=1.42.30
 Requires-Dist: loguru<0.8.0,>=0.7.3
 Requires-Dist: pyyaml<7.0.0,>=6.0.3
 Requires-Dist: requests-toolbelt==1.0.0

dts_dance-0.2.5.dist-info/RECORD

@@ -0,0 +1,15 @@
+dtsdance/__init__.py,sha256=Yl_jEZ5weYfcrklnDvwB4wSgCOvMBLRRgWx0gHs3qfM,49
+dtsdance/bytecloud.py,sha256=s1suo-YexMH6gUHYwC6P5k6AeyOm7Q_acYh_gHCDVjI,5654
+dtsdance/ddutil.py,sha256=aZ-fG_e2ZkHs8sxZsudf6bVCnp_OI2Z4WjbxOfPoJRQ,1134
+dtsdance/dflow.py,sha256=lEn_gd0pUOqEhuvs0hSQICR8LKOk3wP4DBbeyHZ-A44,7766
+dtsdance/dsyncer.py,sha256=gK2rRiO3041P3efbmXgvLPeTgKlJowuPR7CCz_uyoSA,9926
+dtsdance/feishu_base.py,sha256=dYYNnsCjGZYN2I6inR9FRQoUiDLB7j40rzydw50N38o,11825
+dtsdance/feishu_table.py,sha256=JleCjQJ-rcbC15v2UAc5BK-YioB2SMGsdGztQ9BaT4k,10779
+dtsdance/metrics_fe.py,sha256=hzIl5BJmuCrkqJOHELVzXm3YAqrPttbyVkKBglS4mgQ,18978
+dtsdance/s3.py,sha256=Bh-cwLksfO5PewNtIzE_Md3rRLDLI1DUVoOD7Pou5T8,1294
+dtsdance/spacex.py,sha256=oVnHXTj-cuq2vcsUJq33_aaq0TQp7A52kldvspKueI4,4799
+dtsdance/tcc_inner.py,sha256=Z0Wr-z2YPwAti6BO_fsX0VPT5pCbfuCBLQ37UKMFL7U,1629
+dtsdance/tcc_open.py,sha256=JIfVhL3yfm0ftS_vzONE2QNzfqcZuX1mPTvjovAtwmE,6893
+dts_dance-0.2.5.dist-info/METADATA,sha256=-skiMz-CuBlzpt7BBDJpOEXrmsj8np_xKnG-6K_HPao,826
+dts_dance-0.2.5.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+dts_dance-0.2.5.dist-info/RECORD,,

dtsdance/ddutil.py

@@ -0,0 +1,39 @@
+from typing import Any, Optional
+from loguru import logger
+import requests
+
+
+def make_request(method: str, url: str, headers: dict[str, str], payload: Optional[dict] = None) -> dict[str, Any]:
+    """
+    发送 HTTP 请求的通用方法
+
+    Args:
+        method: HTTP 方法 (GET/POST)
+        url: 请求 URL
+        headers: 请求头
+        payload: POST 请求的 JSON 数据
+
+    Returns:
+        dict[str, Any]: 解析后的 JSON 响应
+    """
+    response = None
+    try:
+        if method.upper() == "GET":
+            response = requests.get(url, headers=headers)
+        elif method.upper() == "POST":
+            response = requests.post(url, json=payload, headers=headers)
+        else:
+            raise ValueError(f"不支持的 HTTP 方法: {method}")
+
+        # 检查响应状态码
+        # response.raise_for_status()
+
+        # 解析 JSON 响应
+        return response.json()
+
+    except Exception as e:
+        error_msg = f"make_request occur error, error: {e}"
+        if response is not None:
+            error_msg += f", response.text: {response.text}"
+        logger.warning(error_msg)
+        raise

dtsdance/dflow.py

@@ -1,7 +1,9 @@
-from typing import Any, cast, Optional
+import collections
+from typing import Any, cast
 from loguru import logger
+
+from dtsdance.ddutil import make_request
 from .bytecloud import ByteCloudClient
-import requests
 
 
 class TaskNotFound(Exception):
@@ -17,42 +19,13 @@ class DFlowClient:
     def __init__(self, bytecloud_client: ByteCloudClient) -> None:
         self.bytecloud_client = bytecloud_client
 
-    def _make_request(self, method: str, url: str, headers: dict[str, str], json_data: Optional[dict] = None) -> dict[str, Any]:
-        """
-        发送 HTTP 请求的通用方法
+    def build_request_headers(self, site: str, vregion: str | None) -> dict[str, str]:
+        headers = self.bytecloud_client.build_request_headers(site)
+        if vregion:
+            headers["x-bcgw-vregion"] = vregion
+        return headers
 
-        Args:
-            method: HTTP 方法 (GET/POST)
-            url: 请求 URL
-            headers: 请求头
-            json_data: POST 请求的 JSON 数据
-
-        Returns:
-            dict[str, Any]: 解析后的 JSON 响应
-        """
-        response = None
-        try:
-            if method.upper() == "GET":
-                response = requests.get(url, headers=headers)
-            elif method.upper() == "POST":
-                response = requests.post(url, json=json_data, headers=headers)
-            else:
-                raise ValueError(f"不支持的 HTTP 方法: {method}")
-
-            # 检查响应状态码
-            response.raise_for_status()
-
-            # 解析 JSON 响应
-            return response.json()
-
-        except Exception as e:
-            error_msg = f"_make_request occur error, error: {e}"
-            if response is not None:
-                error_msg += f", response.text: {response.text}"
-            logger.warning(error_msg)
-            raise
-
-    def get_task_info(self, site: str, task_id: str) -> dict[str, Any]:
+    def get_task_info(self, site: str, task_id: str, vregion: str | None = None) -> dict[str, Any]:
         """
         获取 DFlow 任务信息
 
@@ -68,9 +41,9 @@ class DFlowClient:
         url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeTaskInfo"
 
         # 构建请求数据
-        json_data = {"id": int(task_id)}
+        payload = {"id": int(task_id)}
 
-        response_data = self._make_request("POST", url, self.bytecloud_client.build_request_headers(site), json_data)
+        response_data = make_request("POST", url, self.build_request_headers(site, vregion), payload)
 
         message = response_data.get("message")
         # logger.debug(f"get_task_info {site} {task_id}, message: {message}")
@@ -94,7 +67,7 @@ class DFlowClient:
         except (KeyError, AttributeError, Exception) as e:
             raise Exception(f"无法从响应中提取 DFlow 任务信息数据: {str(e)}")
 
-    def get_dflow_info(self, site: str, dflow_id: str) -> dict[str, Any]:
+    def get_dflow_info(self, site: str, dflow_id: str, vregion: str | None = None) -> dict[str, Any]:
         """
         获取 DFlow 进程信息
 
@@ -110,9 +83,9 @@ class DFlowClient:
         url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeDFlowDetail"
 
         # 构建请求数据
-        json_data = {"dflow_id": int(dflow_id)}
+        payload = {"dflow_id": int(dflow_id)}
 
-        response_data = self._make_request("POST", url, self.bytecloud_client.build_request_headers(site), json_data)
+        response_data = make_request("POST", url, self.build_request_headers(site, vregion), payload)
 
         message = response_data.get("message", "")
         # logger.debug(f"get_dflow_info {site} {dflow_id}, message: {message}")
@@ -168,9 +141,9 @@ class DFlowClient:
         url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/InitSystemResource"
 
         # 构建请求数据
-        json_data = {"ctrl_env": ctrl_env}
+        payload = {"ctrl_env": ctrl_env}
 
-        response_data = self._make_request("POST", url, self.bytecloud_client.build_request_headers(site), json_data)
+        response_data = make_request("POST", url, self.build_request_headers(site, None), payload)
 
         message = response_data.get("message")
         logger.info(f"int_resources {site} {ctrl_env}, message: {message}")
@@ -193,9 +166,9 @@ class DFlowClient:
         url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeResources"
 
         # 构建请求数据
-        json_data = {"offset": 0, "limit": 10, "ctrl_env": ctrl_env}
+        payload = {"offset": 0, "limit": 10, "ctrl_env": ctrl_env}
 
-        response_data = self._make_request("POST", url, self.bytecloud_client.build_request_headers(site), json_data)
+        response_data = make_request("POST", url, self.build_request_headers(site, None), payload)
 
         try:
             data = response_data.get("data", {})
@@ -203,3 +176,31 @@ class DFlowClient:
             return [item["name"] for item in items]
         except (KeyError, AttributeError, Exception) as e:
             raise Exception(f"无法从响应中提取 CTRL 环境资源列表数据: {str(e)}")
+
+    def get_all_ctrl_envs(self, site: str) -> dict[str, list[str]]:
+        """获取所有可用的控制环境列表，返回以domain为键，ctrl_env列表为值的字典"""
+        site_info = self.bytecloud_client.get_site_config(site)
+        url = f"{site_info.endpoint}/api/v1/bytedts/api/bytedts/v3/DescribeRegions"
+        resp_json = make_request("POST", url, self.build_request_headers(site, None), {})
+        if resp_json["code"] != 0:
+            raise Exception(resp_json["message"])
+
+        # print(f"Fetched regions info: {json.dumps(resp_json["data"], ensure_ascii=False, indent=2)}")
+
+        # 提取所有控制环境列表，格式为以domain为键，ctrl_env列表为值的字典
+        all_ctrl_envs_by_domain = collections.defaultdict(list[str])
+
+        # 从ops_platform_region中提取ctrl_env_list
+        if "bytedts_env" in resp_json["data"]:
+            for bytedts_env in resp_json["data"]["bytedts_env"]:
+                if "ctrl_env_list" in bytedts_env and "domain" in bytedts_env:
+                    env = bytedts_env["env"]
+                    region = bytedts_env["include_region"][0] if bytedts_env["include_region"] else "Unknown"
+                    env_region_key = env + "|" + region
+                    for ctrl_env_item in bytedts_env["ctrl_env_list"]:
+                        if "ctrl_env" in ctrl_env_item:
+                            ctrl_env = ctrl_env_item["ctrl_env"]
+                            # 将ctrl_env添加到domain对应的列表中
+                            all_ctrl_envs_by_domain[env_region_key].append(ctrl_env)
+
+        return dict(all_ctrl_envs_by_domain)

dtsdance/dsyncer.py

@@ -1,7 +1,7 @@
+from dtsdance.ddutil import make_request
 from .bytecloud import ByteCloudClient
 from typing import Any, NamedTuple, Tuple, cast, Optional
 from loguru import logger
-import requests
 import re
 from datetime import datetime
 
@@ -41,42 +41,6 @@ class DSyncerClient:
 
         return headers
 
-    def _make_request(self, method: str, url: str, headers: dict[str, str], data_raw: Optional[dict] = None) -> dict[str, Any]:
-        """
-        发送 HTTP 请求的通用方法
-
-        Args:
-            method: HTTP 方法 (GET/POST)
-            url: 请求 URL
-            headers: 请求头
-            json_data: POST 请求的 JSON 数据
-            operation_name: 操作名称，用于日志记录
-
-        Returns:
-            dict[str, Any]: 解析后的 JSON 响应
-        """
-        response = None
-        try:
-            if method.upper() == "GET":
-                response = requests.get(url, headers=headers)
-            elif method.upper() == "POST":
-                response = requests.post(url, json=data_raw, headers=headers)
-            else:
-                raise ValueError(f"不支持的 HTTP 方法: {method}")
-
-            # 检查响应状态码
-            # response.raise_for_status()
-
-            # 解析 JSON 响应
-            return response.json()
-
-        except Exception as e:
-            error_msg = f"_make_request occur error, error: {e}"
-            if response is not None:
-                error_msg += f", response.text: {response.text}"
-            logger.warning(error_msg)
-            raise
-
     def _acquire_task_info(self, site: str, task_id: str) -> dict[str, str]:
         """
         获取 DSyncer 任务信息
@@ -95,7 +59,7 @@ class DSyncerClient:
         # 准备请求头
         headers = self._build_headers(site)
 
-        return self._make_request("GET", url, headers)
+        return make_request("GET", url, headers)
 
     def get_dflow_task_info(self, site: str, task_id: str) -> tuple[str, str]:
         """
@@ -108,7 +72,7 @@ class DSyncerClient:
         # 准备请求头
         headers = self._build_headers(site)
 
-        json_data = self._make_request("GET", url, headers)
+        json_data = make_request("GET", url, headers)
 
         message = json_data.get("message", "")
         logger.debug(f"get task migrate info {site} {task_id}, message: {message}")
@@ -214,9 +178,9 @@ class DSyncerClient:
         # 准备请求头
         headers = self._build_headers(site, secret_api=True)
 
-        data_raw = {"task_id": task_id}
+        payload = {"task_id": task_id}
 
-        response_data = self._make_request("POST", url, headers, data_raw)
+        response_data = make_request("POST", url, headers, payload)
 
         message = response_data.get("message", "")
         logger.debug(f"get task migrate status {site} {task_id}, message: {message}")
@@ -236,9 +200,9 @@ class DSyncerClient:
         # 准备请求头
         headers = self._build_headers(site, secret_api=True)
 
-        data_raw = {"task_id_list": [task_id]}
+        payload = {"task_id_list": [task_id]}
 
-        response_data = self._make_request("POST", url, headers, data_raw)
+        response_data = make_request("POST", url, headers, payload)
 
         logger.debug(f"migration_mark_rollback return {site} {task_id}, json_data: {response_data}")
         success_task = response_data.get("data", {}).get("success_task", [])
@@ -255,9 +219,9 @@ class DSyncerClient:
         # 准备请求头
         headers = self._build_headers(site, secret_api=True)
 
-        data_raw = {"task_id": task_id}
+        payload = {"task_id": task_id}
 
-        response_data = self._make_request("POST", url, headers, data_raw)
+        response_data = make_request("POST", url, headers, payload)
 
         logger.debug(f"migration_mark_success return {site} {task_id}, json_data: {response_data}")
         success_task = response_data.get("data", {}).get("success_task", [])
@@ -277,7 +241,7 @@ class DSyncerClient:
         # 准备请求头
         headers = self._build_headers(site, secret_api=True)
 
-        data_raw = {
+        payload = {
             "task_id": task_id,
             "delay_threshold": "20s",
             "dsyncer_task_pause_threshold": "180s",
@@ -289,7 +253,7 @@ class DSyncerClient:
             "app_parallel": app_parallel,
         }
 
-        response_data = self._make_request("POST", url, headers, data_raw)
+        response_data = make_request("POST", url, headers, payload)
 
         logger.debug(f"migrate_task return {site} {task_id}, json_data: {response_data}")
         message = response_data.get("message")

dtsdance/feishu_base.py

@@ -100,13 +100,15 @@ class FeishuBase:
         headers["Authorization"] = f"Bearer {token}"
         kwargs["headers"] = headers
 
+        response: requests.Response | None = None
         try:
             response = requests.request(method, url, **kwargs)
-            logger.debug(f"response_json: {response.json()}")
+            # logger.debug(f"response_json: {response.json()}")
             response.raise_for_status()
             return response
         except requests.RequestException as e:
-            logger.warning(f"API请求失败: {method} {url}, error: {e}")
+            error_detail = response.json() if response is not None else "无响应"
+            logger.warning(f"API请求失败: {method} {url}, error: {e}, response_json: {error_detail}")
             raise
 
     def upload_image(self, image_path: str) -> str | None:
@@ -265,7 +267,7 @@ class FeishuBase:
             logger.warning(f"回复消息失败: {e}")
             raise
 
-    def batch_get_user_ids(self, emails: list[str]) -> list[str]:
+    def batch_get_user_ids(self, emails: list[str]) -> dict[str, None | str]:
         """
         批量获取用户ID
 
@@ -273,7 +275,7 @@ class FeishuBase:
             emails: 用户邮箱列表
 
         Returns:
-            批量获取结果（用户ID列表）
+            批量获取结果（email -> user_id 的字典映射）
 
         Raises:
             Exception: 获取失败
@@ -281,16 +283,51 @@ class FeishuBase:
         payload = {"emails": emails, "include_resigned": False}
 
         try:
-            response = self.make_request("POST", f"/contact/v3/users/batch_get_id?user_id_type=user_id", json=payload)
+            response = self.make_request("POST", "/contact/v3/users/batch_get_id?user_id_type=user_id", json=payload)
             result = response.json()
             if result.get("code") != 0:
                 raise Exception(f"请求失败: {result.get('msg')}")
 
-            user_ids = [user.get("user_id") for user in result.get("data", {}).get("user_list", [])]
+            user_dict = {user.get("email"): user.get("user_id", None) for user in result.get("data", {}).get("user_list", []) if user.get("email")}
 
             logger.debug(f"发送响应logid: {response.headers.get('X-Tt-Logid')}")
 
-            return user_ids
+            return user_dict
         except Exception as e:
             logger.warning(f"批量获取用户ID失败: {e}")
             raise
+
+    def batch_get_all_user_ids(self, emails: list[str]) -> dict[str, None | str]:
+        """
+        批量获取所有用户ID，自动分批处理（每批最多50个）
+
+        Args:
+            emails: 用户邮箱列表
+
+        Returns:
+            所有用户ID字典（email -> user_id 的映射）
+
+        Raises:
+            Exception: 获取失败
+        """
+        if not emails:
+            return {}
+
+        all_user_dict = {}
+        batch_size = 50
+
+        # 分批处理
+        for i in range(0, len(emails), batch_size):
+            batch_emails = emails[i : i + batch_size]
+            logger.info(f"正在获取第 {i // batch_size + 1} 批用户ID，数量: {len(batch_emails)}")
+
+            try:
+                batch_user_dict = self.batch_get_user_ids(batch_emails)
+                all_user_dict.update(batch_user_dict)
+            except Exception as e:
+                logger.error(f"获取第 {i // batch_size + 1} 批用户ID失败: {e}")
+                # 继续处理下一批，不中断整个流程
+                continue
+
+        logger.info(f"共获取到 {len(all_user_dict)} 个用户ID")
+        return all_user_dict

dtsdance/tcc_inner.py

@@ -20,7 +20,7 @@ class TCCInnerClient:
         region: str,
         dir: str,
         conf_name: str,
-    ) -> dict[str, Any]:
+    ) -> list[dict[str, Any]]:
         """
         List TCC configurations.
         """
@@ -49,7 +49,7 @@ class TCCInnerClient:
             if base_resp.get("error_code", 0) != 0:
                 raise TCCError(f"TCC API error: {base_resp.get('error_message', 'Unknown error')}")
 
-            return result.get("data", {})
+            return result.get("data", [])
 
         except requests.RequestException as e:
             raise TCCError(f"Failed to get TCC config: {str(e)}") from e

dtsdance/tcc_open.py

@@ -40,9 +40,9 @@ class TCCClient:
         self,
         ns_name: str,
         region: str,
+        dir: str,
         conf_name: str,
         value: str,
-        dir: str,
         description: str,
         data_type: str,
     ) -> dict[str, Any]:

dts_dance-0.2.4.dist-info/RECORD

@@ -1,14 +0,0 @@
-dtsdance/__init__.py,sha256=Yl_jEZ5weYfcrklnDvwB4wSgCOvMBLRRgWx0gHs3qfM,49
-dtsdance/bytecloud.py,sha256=s1suo-YexMH6gUHYwC6P5k6AeyOm7Q_acYh_gHCDVjI,5654
-dtsdance/dflow.py,sha256=ggwNB2x-LgnpI9nzNTTnYnGfYyGH5rYcHHBhC4pOlfg,7059
-dtsdance/dsyncer.py,sha256=3Oj1Ko5FuB_sUm9Hj-hBy95wee3b6TH2FXKoPFGsakM,11193
-dtsdance/feishu_base.py,sha256=Dsl8CZZ3wyZf3TEiiUAs3_WpX1pinCuWFwqpm29ROVc,10414
-dtsdance/feishu_table.py,sha256=JleCjQJ-rcbC15v2UAc5BK-YioB2SMGsdGztQ9BaT4k,10779
-dtsdance/metrics_fe.py,sha256=hzIl5BJmuCrkqJOHELVzXm3YAqrPttbyVkKBglS4mgQ,18978
-dtsdance/s3.py,sha256=Bh-cwLksfO5PewNtIzE_Md3rRLDLI1DUVoOD7Pou5T8,1294
-dtsdance/spacex_bytedts.py,sha256=oVnHXTj-cuq2vcsUJq33_aaq0TQp7A52kldvspKueI4,4799
-dtsdance/tcc_inner.py,sha256=6Tuq_r2nNCK_HC4hqEGh7UVAa8kDtuIcU95WBu4zuFQ,1623
-dtsdance/tcc_open.py,sha256=Qb_ue3xH0CSsoReItdFKuvW7JsDTfFkDNIB_4zjzSQI,6893
-dts_dance-0.2.4.dist-info/METADATA,sha256=LY_7WWh06QNmhr_1vyuMTtKDrx88N0P1BjGEURYAl6U,833
-dts_dance-0.2.4.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-dts_dance-0.2.4.dist-info/RECORD,,