mlflow-tclake-plugin 2.1.7.dev2__tar.gz → 2.1.7.dev4__tar.gz

{mlflow_tclake_plugin-2.1.7.dev2 → mlflow_tclake_plugin-2.1.7.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlflow-tclake-plugin
-Version: 2.1.7.dev2
+Version: 2.1.7.dev4
 Summary: Tclake plugin for MLflow
 License-File: LICENSE.txt
 Requires-Dist: mlflow==2.17.2

mlflow_tclake_plugin-2.1.7.dev4/mlflow_tclake_plugin/credential_provider.py

@@ -0,0 +1,466 @@
+"""
+腾讯云凭证提供器（自包含，不依赖 wedata_automl）
+
+密钥优先级（高 -> 低）：
+
+    1. 固定密钥：`TENCENTCLOUD_SECRET_ID` + `TENCENTCLOUD_SECRET_KEY` 且**没有**
+       `TENCENTCLOUD_TOKEN` —— 视为长期密钥，直接使用，不做续期、不启动定时任务。
+    2. 临时密钥：以环境变量中的密钥（含 `TENCENTCLOUD_TOKEN`）作为引导凭证，
+       用 `CommonClient` 调用 wedata 云 API `DescribeCamTemporaryToken` 换取临时密钥，
+       并由后台定时任务在**过期前 60 秒**自动续期。
+
+为什么续期必须用定时任务，而不是"用的时候再检查"
+--------------------------------------------------
+续期本身也是一次需要签名的云 API 调用，用的是当前这把临时密钥。一旦它彻底过期，
+续期请求会因鉴权失败而无法完成，凭证链就永久断裂。所以必须在过期前主动续期 ——
+即使这段时间内业务侧完全没有调用云 API。
+
+ProductName 说明
+----------------
+`DescribeCamTemporaryToken` 的 `ProductName` 仅支持 `wedata` 与 `tccatalog`，
+服务端据此下发 `name/{ProductName}:*` 的授权策略，两者的 token 不能互用。
+本插件访问 tccatalog，因此固定使用 `tccatalog`。
+
+注意：`DescribeCamTemporaryToken` 本身始终发往 **wedata** 服务端点
+（`TENCENTCLOUD_ENDPOINT` 指向的是 tccatalog，不能用于这个调用）。
+
+注意：本模块刻意不依赖 `wedata_automl`——本插件是被 wedata_automl 安装使用的，
+反向依赖会形成循环。
+"""
+
+import os
+import threading
+import time
+
+ENV_SECRET_ID = "TENCENTCLOUD_SECRET_ID"
+ENV_SECRET_KEY = "TENCENTCLOUD_SECRET_KEY"
+ENV_TOKEN = "TENCENTCLOUD_TOKEN"
+ENV_TMP_TOKEN = "TENCENTCLOUD_TMP_TOKEN"
+ENV_PROJECT_ID = "WEDATA_PROJECT_ID"
+ENV_WEDATA_ENDPOINT = "TENCENT_CLOUD_SDK_ENDPOINT"
+ENV_REFRESH_AHEAD = "WEDATA_SECRET_REFRESH_AHEAD_SECONDS"
+ENV_DISABLE_DYNAMIC = "WEDATA_DISABLE_DYNAMIC_SECRET"
+
+PRODUCT_WEDATA = "wedata"
+PRODUCT_TCCATALOG = "tccatalog"
+
+WEDATA_SERVICE = "wedata"
+WEDATA_API_VERSION = "2021-08-20"
+DEFAULT_WEDATA_ENDPOINT = "wedata.internal.tencentcloudapi.com"
+
+DEFAULT_REFRESH_AHEAD_SECONDS = 60
+MIN_REFRESH_DELAY_SECONDS = 5
+RETRY_DELAY_SECONDS = 30
+# ExpiredTime 合法区间（秒级时间戳）：2020-01-01 ~ 2100-01-01
+MIN_VALID_EPOCH_SECONDS = 1577836800
+MAX_VALID_EPOCH_SECONDS = 4102444800
+
+
+def _debug(msg):
+    if os.getenv("TENCENTCLOUD_DEBUG", None):
+        print(msg)
+
+
+def mask_secret_id(secret_id):
+    """脱敏展示 SecretId，仅保留前后各 4 位"""
+    if not secret_id:
+        return ""
+    if len(secret_id) <= 8:
+        return "*" * len(secret_id)
+    return "{}{}{}".format(secret_id[:4], "*" * 6, secret_id[-4:])
+
+
+def _read_refresh_ahead():
+    try:
+        value = int(os.getenv(ENV_REFRESH_AHEAD, str(DEFAULT_REFRESH_AHEAD_SECONDS)))
+        return value if value >= 0 else DEFAULT_REFRESH_AHEAD_SECONDS
+    except (TypeError, ValueError):
+        return DEFAULT_REFRESH_AHEAD_SECONDS
+
+
+def _dynamic_disabled():
+    """是否禁用"调云 API 换临时密钥"（逃生阀）"""
+    return str(os.getenv(ENV_DISABLE_DYNAMIC, "")).lower() in ("1", "true", "yes")
+
+
+def _parse_expire(value):
+    """
+    解析 ExpiredTime
+
+    `DescribeCamTemporaryToken` 返回的 `ExpiredTime` 是**秒级绝对时间戳**
+    （例如 1788789863 -> 2026-09-07 22:04:23）。
+
+    只做解析与合法性校验，不做语义猜测：异常值告警并返回 0，
+    由上层按"没有到期信息"处理，避免静默换算把问题藏起来。
+    """
+    if value is None:
+        return 0
+    try:
+        expire = int(value)
+    except (TypeError, ValueError):
+        _debug("invalid ExpiredTime (not an int): {!r}".format(value))
+        return 0
+    if expire <= 0:
+        _debug("invalid ExpiredTime (non-positive): {}".format(expire))
+        return 0
+    if expire < MIN_VALID_EPOCH_SECONDS or expire > MAX_VALID_EPOCH_SECONDS:
+        _debug("ExpiredTime {} is not a plausible epoch-seconds value; "
+               "treating as unknown".format(expire))
+        return 0
+    return expire
+
+
+class TempCredential(object):
+    """密钥快照"""
+
+    __slots__ = ("secret_id", "secret_key", "token", "expire", "owner_uin", "user_id")
+
+    def __init__(self, secret_id=None, secret_key=None, token=None, expire=0,
+                 owner_uin=None, user_id=None):
+        self.secret_id = secret_id or None
+        self.secret_key = secret_key or None
+        self.token = token or None
+        self.expire = int(expire or 0)
+        self.owner_uin = owner_uin or None
+        self.user_id = user_id or None
+
+    @property
+    def valid(self):
+        return bool(self.secret_id and self.secret_key)
+
+    @property
+    def is_static(self):
+        """无 token 即视为固定（长期）密钥"""
+        return self.valid and not self.token
+
+    def identity(self):
+        return self.secret_id, self.secret_key, self.token, self.expire
+
+    def is_expiring(self, refresh_ahead_seconds=DEFAULT_REFRESH_AHEAD_SECONDS):
+        if self.expire <= 0:
+            return False
+        return time.time() >= (self.expire - max(refresh_ahead_seconds, 0))
+
+    def seconds_to_expire(self):
+        if self.expire <= 0:
+            return None
+        return int(self.expire - time.time())
+
+    def __repr__(self):  # 不输出 SecretKey / Token
+        return "TempCredential(secret_id={}, has_token={}, expire={}, ttl={})".format(
+            mask_secret_id(self.secret_id), bool(self.token), self.expire,
+            self.seconds_to_expire()
+        )
+
+
+class CredentialProvider(object):
+    """
+    凭证提供器
+
+    - 固定密钥：直接返回，不启动定时任务
+    - 临时密钥：调用 `DescribeCamTemporaryToken` 换取，并由后台定时任务提前续期
+    """
+
+    def __init__(self, product_name=PRODUCT_TCCATALOG, project_id=None, region=None,
+                 refresh_ahead_seconds=None):
+        self._product_name = product_name or PRODUCT_TCCATALOG
+        self._project_id = project_id or os.getenv(ENV_PROJECT_ID) or None
+        self._region = region
+        self._refresh_ahead_seconds = (
+            refresh_ahead_seconds if refresh_ahead_seconds is not None else _read_refresh_ahead()
+        )
+        self._base = TempCredential(
+            secret_id=os.getenv(ENV_SECRET_ID) or None,
+            secret_key=os.getenv(ENV_SECRET_KEY) or None,
+            token=os.getenv(ENV_TOKEN) or os.getenv(ENV_TMP_TOKEN) or None,
+            expire=0,
+        )
+        self._cached = None
+        self._timer = None
+        self._stopped = False
+        self._lock = threading.RLock()
+
+    @property
+    def product_name(self):
+        return self._product_name
+
+    @property
+    def use_static_secret(self):
+        """固定密钥（无 token），或逃生阀开启 -> 不调云 API 换密钥"""
+        return self._base.is_static or _dynamic_disabled()
+
+    def get(self, force_refresh=False):
+        """获取当前可用密钥；临时密钥在过期/即将过期时同步续期一次（兜底）"""
+        if self.use_static_secret:
+            return self._base
+
+        with self._lock:
+            cached = self._cached
+            if (
+                not force_refresh
+                and cached is not None
+                and cached.valid
+                and not cached.is_expiring(self._refresh_ahead_seconds)
+            ):
+                return cached
+
+            refreshed = self._refresh_locked()
+            if refreshed is not None:
+                return refreshed
+            if cached is not None and cached.valid:
+                return cached
+            return self._base
+
+    def start_auto_refresh(self):
+        """启动后台定时续期任务（幂等）；固定密钥场景不创建线程"""
+        if self.use_static_secret:
+            return False
+
+        with self._lock:
+            self._stopped = False
+            if self._timer is not None:
+                return True
+            self._refresh_locked()
+            self._schedule_locked()
+            return True
+
+    def stop_auto_refresh(self):
+        with self._lock:
+            self._stopped = True
+            if self._timer is not None:
+                self._timer.cancel()
+                self._timer = None
+
+    def _schedule_locked(self):
+        if self._stopped:
+            return
+        cached = self._cached
+        if cached is not None and cached.valid and cached.expire > 0:
+            ttl = cached.expire - time.time()
+            delay = ttl - self._refresh_ahead_seconds
+            if delay < MIN_REFRESH_DELAY_SECONDS:
+                # TTL 比提前量还短：固定用最小延迟会变成每 5s 打一次云 API（可能限频），
+                # 退化为按 TTL 的一半续期
+                delay = max(ttl / 2.0, MIN_REFRESH_DELAY_SECONDS)
+                _debug("credential ttl ({}s) shorter than refresh-ahead ({}s), "
+                       "next refresh in {}s".format(
+                           int(ttl), self._refresh_ahead_seconds, int(delay)))
+        else:
+            delay = RETRY_DELAY_SECONDS
+
+        timer = threading.Timer(delay, self._on_timer)
+        timer.daemon = True
+        timer.name = "tclake-cred-refresh"
+        timer.start()
+        self._timer = timer
+        _debug("next credential refresh in {}s".format(int(delay)))
+
+    def _on_timer(self):
+        with self._lock:
+            self._timer = None
+            if self._stopped:
+                return
+            try:
+                self._refresh_locked()
+            except Exception as e:
+                _debug("scheduled credential refresh failed: {}".format(e))
+            self._schedule_locked()
+
+    def _refresh_locked(self):
+        fetched = self._fetch_cam_token()
+        if fetched is None or not fetched.valid:
+            return None
+        changed = self._cached is None or self._cached.identity() != fetched.identity()
+        self._cached = fetched
+        self.sync_tencentcloud_env(fetched)
+        if changed:
+            _debug("cam temporary credential refreshed: {}".format(fetched))
+        return fetched
+
+    def _signing_credential(self):
+        """构造调用 DescribeCamTemporaryToken 的签名凭证（链式续期）"""
+        from tencentcloud.common import credential as tc_credential
+
+        cached = self._cached
+        if cached is not None and cached.valid and not cached.is_expiring(0):
+            source = cached
+        else:
+            source = self._base
+        return tc_credential.Credential(source.secret_id, source.secret_key, source.token)
+
+    def _fetch_cam_token(self):
+        if not self._base.valid:
+            _debug("cannot fetch cam temporary token: secret id/key not set")
+            return None
+        if not self._project_id:
+            _debug("cannot fetch cam temporary token: WEDATA_PROJECT_ID not set")
+            return None
+        if not self._region:
+            _debug("cannot fetch cam temporary token: region unknown")
+            return None
+
+        try:
+            from tencentcloud.common.common_client import CommonClient
+
+            client = CommonClient(
+                WEDATA_SERVICE,
+                WEDATA_API_VERSION,
+                self._signing_credential(),
+                self._region,
+                self._build_wedata_profile(),
+            )
+            params = {"ProjectId": self._project_id, "ProductName": self._product_name}
+            resp = client.call_json("DescribeCamTemporaryToken", params)
+        except Exception as e:
+            _debug("call DescribeCamTemporaryToken failed: {}".format(e))
+            return None
+
+        data = (resp or {}).get("Response", {}).get("Data") or {}
+        secret_id = data.get("SecretId")
+        secret_key = data.get("SecretKey")
+        if not secret_id or not secret_key:
+            _debug("DescribeCamTemporaryToken returned no credential")
+            return None
+
+        return TempCredential(
+            secret_id=secret_id,
+            secret_key=secret_key,
+            token=data.get("Token"),
+            expire=_normalize_expire(data.get("ExpiredTime")),
+            owner_uin=data.get("OwnerUserId"),
+            user_id=data.get("UserId"),
+        )
+
+    @staticmethod
+    def _build_wedata_profile():
+        """
+        构造调用 wedata 云 API 的 ClientProfile
+
+        不能复用 TENCENTCLOUD_ENDPOINT——那个指向 tccatalog。
+        """
+        from tencentcloud.common.profile.client_profile import ClientProfile
+        from tencentcloud.common.profile.http_profile import HttpProfile
+
+        http_profile = HttpProfile()
+        http_profile.protocol = "https"
+        http_profile.endpoint = os.getenv(ENV_WEDATA_ENDPOINT) or DEFAULT_WEDATA_ENDPOINT
+        client_profile = ClientProfile()
+        client_profile.httpProfile = http_profile
+        return client_profile
+
+    @staticmethod
+    def sync_tencentcloud_env(cred):
+        """把最新密钥同步到 TENCENTCLOUD_* 环境变量"""
+        if not cred.valid:
+            return
+        os.environ[ENV_SECRET_ID] = cred.secret_id
+        os.environ[ENV_SECRET_KEY] = cred.secret_key
+        if cred.token:
+            os.environ[ENV_TOKEN] = cred.token
+            os.environ[ENV_TMP_TOKEN] = cred.token
+
+    def as_sdk_credential(self):
+        """包装为腾讯云 SDK 可直接使用的动态凭证"""
+        return DynamicCredential(self)
+
+
+class DynamicCredential(object):
+    """
+    腾讯云 SDK 动态凭证
+
+    SDK 每次请求签名（含每次重试）都会调用 `get_credential_info()`，
+    因此 client 无需重建即可用上续期后的密钥。
+    """
+
+    def __init__(self, provider):
+        self._provider = provider
+
+    def _current(self):
+        cred = self._provider.get()
+        if not cred.valid:
+            raise _credential_error(
+                "no valid tencentcloud credential: {}/{} not set and "
+                "DescribeCamTemporaryToken unavailable".format(ENV_SECRET_ID, ENV_SECRET_KEY)
+            )
+        return cred
+
+    def get_credential_info(self):
+        cred = self._current()
+        return cred.secret_id, cred.secret_key, cred.token
+
+    @property
+    def secret_id(self):
+        return self._current().secret_id
+
+    @property
+    def secretId(self):
+        return self.secret_id
+
+    @property
+    def secret_key(self):
+        return self._current().secret_key
+
+    @property
+    def secretKey(self):
+        return self.secret_key
+
+    @property
+    def token(self):
+        return self._current().token
+
+    @property
+    def provider(self):
+        return self._provider
+
+    def __repr__(self):  # 不输出 SecretKey / Token
+        return "DynamicCredential(product={}, static={})".format(
+            self._provider.product_name, self._provider.use_static_secret
+        )
+
+
+def _credential_error(message):
+    try:
+        from mlflow.exceptions import MlflowException
+
+        return MlflowException(message)
+    except ImportError:
+        return ValueError(message)
+
+
+_providers = {}
+_providers_lock = threading.Lock()
+
+
+def get_default_provider(product_name=PRODUCT_TCCATALOG, project_id=None, region=None,
+                         auto_refresh=True):
+    """
+    获取（并按需创建）进程级共享的 Provider
+
+    按 (product_name, project_id, region) 维度缓存：wedata 与 tccatalog 的
+    token 授权范围不同，不能共用。
+    """
+    key = (
+        product_name or PRODUCT_TCCATALOG,
+        project_id or os.getenv(ENV_PROJECT_ID) or None,
+        region,
+    )
+    with _providers_lock:
+        provider = _providers.get(key)
+        if provider is None:
+            provider = CredentialProvider(
+                product_name=key[0], project_id=key[1], region=key[2]
+            )
+            _providers[key] = provider
+    if auto_refresh:
+        provider.start_auto_refresh()
+    return provider
+
+
+def reset_providers():
+    """停止并清空所有 Provider（主要用于测试）"""
+    global _providers
+    with _providers_lock:
+        providers = list(_providers.values())
+        _providers = {}
+    for provider in providers:
+        provider.stop_auto_refresh()

{mlflow_tclake_plugin-2.1.7.dev2 → mlflow_tclake_plugin-2.1.7.dev4}/mlflow_tclake_plugin/tclake_store.py

@@ -6,10 +6,11 @@ import uuid
 from datetime import datetime
 
 from cachetools import TTLCache
-from tencentcloud.common import credential
 from tencentcloud.common.common_client import CommonClient
 from tencentcloud.common.profile.client_profile import ClientProfile
 
+from mlflow_tclake_plugin.credential_provider import PRODUCT_TCCATALOG, get_default_provider
+
 from mlflow.entities.model_registry import RegisteredModel, ModelVersion, ModelVersionTag
 from mlflow.exceptions import MlflowException
 from mlflow.protos.databricks_pb2 import INVALID_PARAMETER_VALUE, RESOURCE_ALREADY_EXISTS
@@ -302,19 +303,24 @@ class TCLakeStore(AbstractStore):
     def __init__(self, store_uri=None, tracking_uri=None):
         super().__init__(store_uri, tracking_uri)
         log_msg("initializing tencent tclake client {} {}".format(store_uri, tracking_uri))
-        sid = os.getenv("TENCENTCLOUD_SECRET_ID", "")
-        if len(sid) == 0:
-            raise MlflowException("TENCENTCLOUD_SECRET_ID is not set")
-        sk = os.getenv("TENCENTCLOUD_SECRET_KEY", "")
-        if len(sk) == 0:
-            raise MlflowException("TENCENTCLOUD_SECRET_KEY is not set")
-        token = os.getenv("TENCENTCLOUD_TOKEN", None)
-        client_profile = get_tencent_cloud_client_profile()
-        cred = credential.Credential(sid, sk, token)
-        parts = store_uri.split(":")
+        parts = (store_uri or "").split(":")
         if len(parts) < 2:
             raise MlflowException("set store_uri tclake:{region}")
         region = parts[1]
+        # 凭证优先级：固定密钥 > DescribeCamTemporaryToken 换取的临时密钥。
+        # 初始化即启动定时续期（过期前 60s），并包装成动态凭证——
+        # SDK 每次请求签名都会重新取值，client 无需重建。
+        self.credential_provider = get_default_provider(
+            product_name=PRODUCT_TCCATALOG, region=region
+        )
+        self.credential_provider.start_auto_refresh()
+        cloud_credential = self.credential_provider.get()
+        if not cloud_credential.secret_id:
+            raise MlflowException("TENCENTCLOUD_SECRET_ID is not set")
+        if not cloud_credential.secret_key:
+            raise MlflowException("TENCENTCLOUD_SECRET_KEY is not set")
+        client_profile = get_tencent_cloud_client_profile()
+        cred = self.credential_provider.as_sdk_credential()
         self.client = CommonClient("tccatalog", "2024-10-24", cred, region, client_profile)
         self.headers = get_tencent_cloud_headers()
         self.default_catalog_name = os.getenv("TENCENTCLOUD_DEFAULT_CATALOG_NAME", "default")

{mlflow_tclake_plugin-2.1.7.dev2 → mlflow_tclake_plugin-2.1.7.dev4}/mlflow_tclake_plugin.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mlflow-tclake-plugin
-Version: 2.1.7.dev2
+Version: 2.1.7.dev4
 Summary: Tclake plugin for MLflow
 License-File: LICENSE.txt
 Requires-Dist: mlflow==2.17.2

{mlflow_tclake_plugin-2.1.7.dev2 → mlflow_tclake_plugin-2.1.7.dev4}/mlflow_tclake_plugin.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ LICENSE.txt
 README.md
 setup.py
 mlflow_tclake_plugin/__init__.py
+mlflow_tclake_plugin/credential_provider.py
 mlflow_tclake_plugin/tclake_store.py
 mlflow_tclake_plugin.egg-info/PKG-INFO
 mlflow_tclake_plugin.egg-info/SOURCES.txt

{mlflow_tclake_plugin-2.1.7.dev2 → mlflow_tclake_plugin-2.1.7.dev4}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="mlflow-tclake-plugin",
-    version="2.1.7.dev2",
+    version="2.1.7.dev4",
     description="Tclake plugin for MLflow",
     packages=find_packages(),
     # Require MLflow as a dependency of the plugin, so that plugin users can simply install