wedata3-feature-engineering 0.2.6.dev2__tar.gz → 0.2.6.dev4__tar.gz

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/PKG-INFO

@@ -1,41 +1,20 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: wedata3-feature-engineering
-Version: 0.2.6.dev2
+Version: 0.2.6.dev4
 Summary: Wedata Feature Engineering Library Development
-Home-page: 
+Home-page: UNKNOWN
 Author: meahqian
 Author-email: 
 License: Apache 2.0
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: pandas>=1.0.0
-Requires-Dist: feast[postgres,redis]==0.49.0
-Requires-Dist: grpcio==1.74.0
-Requires-Dist: msgpack<2,>=1.0
-Requires-Dist: tencentcloud-sdk-python
-Requires-Dist: ipython
-Requires-Dist: msgpack<2.0.0,>=1.0.0
 Provides-Extra: mlflow2
-Requires-Dist: mlflow==2.17.2; extra == "mlflow2"
 Provides-Extra: mlflow3
-Requires-Dist: mlflow<3.11.0,>=3.10.0; extra == "mlflow3"
 Provides-Extra: dev
-Requires-Dist: flake8; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: python-dotenv; extra == "dev"
-Requires-Dist: fakeredis[lua]; extra == "dev"
-Dynamic: author
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: license
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
 
 # wedata3-feature-engineering
 
@@ -160,3 +139,5 @@ bash build.sh
 ## 许可证
 
 Apache License 2.0
+
+

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/wedata/common/cloud_sdk_client/models.py

@@ -1184,7 +1184,7 @@ class CreateOnlineTableRequest(AbstractModel):
         TargetPath: 目标路径
         OfflineFeatureConfiguration: 离线表信息
         TaskSchedulerConfiguration: 任务调度配置
-        OnlineStoreType: 在线存储类型（TENCENT_POSTGRESQL | REDIS，缺省 TENCENT_POSTGRESQL 保证旧调用方兼容）
+        OnlineStoreType: 在线存储类型（TENCENT_POSTGRE | REDIS，缺省 TENCENT_POSTGRE）
     """
 
     def __init__(self) -> None:
@@ -1351,7 +1351,7 @@ class DeleteOnlineTableRequest(AbstractModel):
         WorkspaceId: 工作空间ID
         TableNameInfo: 表信息
         ResourceGroupId: 资源组ID
-        OnlineStoreType: 在线存储类型（TENCENT_POSTGRESQL | REDIS，缺省 TENCENT_POSTGRESQL 保证旧调用方兼容）
+        OnlineStoreType: 在线存储类型（TENCENT_POSTGRE | REDIS，缺省 TENCENT_POSTGRE）
     """
     def __init__(self) -> None:
         self._WorkspaceId = None

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/wedata/common/constants/constants.py

@@ -46,7 +46,7 @@ FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "wedata3-feature-engineering"
 # 特征查找版本号
 # FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.6.16"
 # FEATURE_LOOKUP_CLIENT_PIP_PACKAGE_VERSION = f"{FEATURE_LOOKUP_CLIENT_PIP_PACKAGE}>=0.6.16"
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.2.6.dev2"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.2.6.dev4"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE_VERSION = f"{FEATURE_LOOKUP_CLIENT_PIP_PACKAGE}>=0.2.4"
 
 # 特征存储内部数据目录

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/wedata/common/online_store/packed_codec.py

@@ -50,6 +50,29 @@ _DATA_VERSION_RAND_LEN = 2
 _OVERSIZE_WARNED: set = set()
 
 
+def _check_msgpack_impl() -> None:
+    """检测 msgpack 是否走 C 扩展实现（方案 D3.1.1 落地约束：防静默降级）。
+
+    zipimport 等场景下 msgpack 会静默退回纯 Python 实现，5 亿行同步慢数十倍。
+    Executor 上通过 ``spark.executorEnv.PYTHONPATH=./deps`` 分发的归档若平台 ABI
+    不匹配即触发该情况，必须在运行时告警而不是等同步超时才发现。
+    """
+    try:
+        import msgpack
+
+        if "fallback" in str(msgpack.Packer):
+            logger.warning(
+                "msgpack is running in PURE-PYTHON fallback mode (C extension not loaded); "
+                "redis packed sync will be extremely slow. Check that the executor deps archive "
+                "matches the executor platform/ABI (see scripts/build_executor_deps.sh)."
+            )
+    except Exception:  # pragma: no cover - 检测本身不允许影响主流程
+        logger.warning("msgpack implementation check skipped (import failed)", exc_info=True)
+
+
+_check_msgpack_impl()
+
+
 class PackedCodecError(ValueError):
     """编解码层的显式失败（对应 codec 设计 §8 的 F2/F3/F8 等）。"""
 
@@ -389,19 +412,58 @@ def _encode_value(vtype: str, v: Any) -> Any:
         return None
     if isinstance(v, float) and math.isnan(v):
         return None
-    if vtype == "UNIX_TIMESTAMP" and isinstance(v, _dt.datetime):
+    if isinstance(v, Decimal):
+        return _encode_decimal(vtype, v)
+    if isinstance(v, _dt.date):
+        return _encode_temporal(vtype, v)
+    return v
+
+
+def _encode_decimal(vtype: str, v: Decimal) -> Any:
+    """Spark DecimalType → Decimal。Java 侧把 decimal 映射为 STRING（未知类型降级），msgpack 不能直接序列化 Decimal。"""
+    if not v.is_finite():
+        return None
+    if vtype == "STRING":
+        return format(v, "f")  # 保留小数位、不出现科学计数法（Decimal('1E+2') → '100'）
+    if vtype in ("DOUBLE", "FLOAT"):
+        return float(v)
+    if vtype in ("INT32", "INT64"):
+        if v != v.to_integral_value():
+            raise PackedCodecError(f"小数 {v} 不能写入整数类型 {vtype}")
+        return int(v)
+    raise PackedCodecError(f"Decimal 不能写入 {vtype} 类型")
+
+
+def _encode_temporal(vtype: str, v: _dt.date) -> Any:
+    """datetime / date。UNIX_TIMESTAMP 存 epoch 秒；Java 把 date 映射为 STRING，按 ISO 格式存。"""
+    if vtype == "UNIX_TIMESTAMP":
+        if not isinstance(v, _dt.datetime):
+            v = _dt.datetime(v.year, v.month, v.day)  # date 按当天 0 点
         if v.tzinfo is None:
             v = v.replace(tzinfo=_dt.timezone.utc)
         return int(v.timestamp())
-    return v
+    if vtype == "STRING":
+        return v.isoformat()
+    raise PackedCodecError(f"{type(v).__name__} 不能写入 {vtype} 类型")
 
 
 def encode_row(meta: SchemaMeta, ts_micros: Optional[int], values: Mapping[str, Any]) -> bytes:
     """dict → 位置数组 msgpack；缺失字段编码为 nil。下标 0 固定为 event_ts_micros。"""
     arr: List[Any] = [None if ts_micros is None else int(ts_micros)]
     for name, vtype in meta.fields:
-        arr.append(_encode_value(vtype, values.get(name)))
-    return msgpack.packb(arr, use_bin_type=True)
+        try:
+            arr.append(_encode_value(vtype, values.get(name)))
+        except PackedCodecError as e:
+            raise PackedCodecError(f"特征 {name} 编码失败: {e}") from None
+    try:
+        return msgpack.packb(arr, use_bin_type=True)
+    except TypeError as e:
+        # 兜底：出现 codec 没覆盖的类型时报清楚是哪一列，而不是裸 msgpack 错误
+        bad = [n for (n, _), x in zip(meta.fields, arr[1:]) if not isinstance(x, _MSGPACK_SCALARS)]
+        raise PackedCodecError(f"特征 {bad} 的值类型不支持序列化: {e}") from None
+
+
+_MSGPACK_SCALARS = (type(None), bool, int, float, str, bytes, bytearray)
 
 
 def decode_row(
@@ -617,20 +679,35 @@ ACTIVATE_LUA = r"""
 -- 返回 {switched(1/0), previous_active or "", reason}
 local new_member = ARGV[1]
 local now = tonumber(ARGV[2])
+-- 水位超过 now + 1 天视为脏数据（未来时间戳），按 0 参与比较：
+-- 否则一条脏数据会让之后所有批次都被判旧（与 WATERMARK_FUTURE_TOLERANCE_SECONDS 一致）
+local MAX_FUTURE_SECONDS = 86400
+local function watermark_of(info)
+  if info.watermark > now + MAX_FUTURE_SECONDS then
+    return 0
+  end
+  return info.watermark
+end
 
 local new_raw = redis.call('HGET', KEYS[2], new_member)
 if not new_raw then
   return {0, '', 'NEW_NOT_REGISTERED'}
 end
 local new_info = cjson.decode(new_raw)
+-- 只激活写入中的批次：超时被判 ABANDONED、已 REJECTED / PURGED 的批次数据可能已被清理，激活即全量 miss
+if new_info.state ~= 'WRITING' then
+  return {0, redis.call('GET', KEYS[1]) or '', 'NOT_WRITING'}
+end
 
 local cur_member = redis.call('GET', KEYS[1])
 if cur_member then
   local cur_raw = redis.call('HGET', KEYS[2], cur_member)
   if cur_raw then
     local cur = cjson.decode(cur_raw)
-    local newer = (new_info.watermark > cur.watermark) or
-                  (new_info.watermark == cur.watermark and new_info.run_start > cur.run_start)
+    local new_wm = watermark_of(new_info)
+    local cur_wm = watermark_of(cur)
+    local newer = (new_wm > cur_wm) or
+                  (new_wm == cur_wm and new_info.run_start > cur.run_start)
     if not newer then
       new_info.state = 'STALE'
       new_info.updated_at = now
@@ -651,3 +728,7 @@ new_info.updated_at = now
 redis.call('HSET', KEYS[2], new_member, cjson.encode(new_info))
 return {1, cur_member or '', 'SWITCHED'}
 """
+
+#: 批次水位允许超出当前时间的上限（秒）。同步模板登记时超出即截断为当前时间并告警，ACTIVATE_LUA 比较时超出按 0 处理。
+#: 1 天能容纳时区错配（P1-13 的 8 小时偏移）与机器时钟偏差。
+WATERMARK_FUTURE_TOLERANCE_SECONDS = 86400

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/wedata/feature_engineering/client.py

@@ -1095,7 +1095,7 @@ class FeatureEngineeringClient:
                 由用户根据所在工作空间已配置的在线 catalog 显式指定（如 `DataLakeCatalog`），
                 SDK 不提供默认兜底以避免在多 catalog 场景下静默连接错误的 PG 实例。Redis 不传
             trigger: [可选]周期性任务的调度配置
-            online_store_type: [可选]"TENCENT_POSTGRESQL"（默认）或 "REDIS"。Redis 在线表由离线表与
+            online_store_type: [可选]"TENCENT_POSTGRE"（默认）或 "REDIS"。Redis 在线表由离线表与
                 工作空间的 Redis 数据源确定，不需要也不接受在线库表名
         """
         kwargs = {}

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/wedata/feature_engineering/feast/infra/online_store/cleanup_packed.py

@@ -15,6 +15,7 @@ key 格式与 BatchInfo 编解码统一走 ``wedata.common.online_store.packed_c
 | 状态 | 处置 |
 |---|---|
 | WRITING（未超时 <6h） | 不清理（同步进行中） |
+| WRITING（作业失败收尾）| ``reject_batch`` 置 REJECTED 并立即清理，不等 6h |
 | WRITING（超时 ≥6h）| 转 ABANDONED 后立即清理 |
 | ACTIVE | **永不清理**（二次确认防误删） |
 | RETIRED（最新 1 个，retired <1h） | 保留（回滚窗口） |
@@ -30,6 +31,7 @@ from __future__ import annotations
 
 import logging
 import re
+import threading
 import time
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -134,14 +136,15 @@ def cleanup(client: redis.Redis, prefix: str) -> int:
         if state in (pc.STATE_REJECTED, pc.STATE_STALE, pc.STATE_ABANDONED):
             to_delete.append((member, info))
         elif state == pc.STATE_WRITING and _is_abandoned(info, now):
-            _mark_state(client, prefix, member, pc.STATE_ABANDONED, now)
-            to_delete.append((member, info))
+            # 原子判定：期间批次被激活或续了心跳就不动它（快照判超时、再非原子改状态会把刚激活的批次删掉）
+            if _mark_abandoned(client, prefix, member, info, now):
+                to_delete.append((member, info))
         # ACTIVE / 未超时 WRITING / PURGED：不删数据
 
     deleted = 0
     for member, _info in to_delete:
-        if member == active_member:
-            # 防御性检查：理论不可能（ACTIVE 不在 to_delete），双重保险
+        # active 以删除前实时读取为准，不用上面的快照：快照之后可能已切换
+        if member in (active_member, _get_active_member(client, prefix)):
             logger.warning("skip purge of active batch %r on %r", member, prefix)
             continue
         try:
@@ -170,6 +173,149 @@ def purge_batch(client: redis.Redis, prefix: str, member: str) -> None:
     _mark_state(client, prefix, member, pc.STATE_PURGED, time.time())
 
 
+#: 只把仍处于 WRITING 的批次改为 REJECTED，返回改写后的状态（批次不存在返回空串）。
+#: 原子判断：激活脚本已执行但客户端没收到回包时，批次可能已是 ACTIVE，绝不能被改写。
+_REJECT_LUA = r"""
+local raw = redis.call('HGET', KEYS[1], ARGV[1])
+if not raw then
+  return ''
+end
+local info = cjson.decode(raw)
+if info.state == 'WRITING' then
+  info.state = 'REJECTED'
+  info.updated_at = tonumber(ARGV[2])
+  info.reason = ARGV[3]
+  redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(info))
+end
+return info.state
+"""
+
+#: 失败收尾时可以立即删除数据的状态（REJECTED 来自本函数，STALE 来自激活脚本判旧）
+_DISCARDABLE_STATES = frozenset({pc.STATE_REJECTED, pc.STATE_STALE})
+
+
+def reject_batch(client: redis.Redis, prefix: str, member: str, reason: str) -> bool:
+    """
+    同步作业失败收尾：本批 WRITING → REJECTED，并立即删除本批数据，返回是否删除了数据。
+
+    不收尾的话批次停在 WRITING，下一次同步的门禁要等 ``MAX_RUN_HOURS`` 才放行；
+    激活被判旧（STALE）的批次同样立即删除，不留一份全量数据占内存到下一次同步。
+    已激活（ACTIVE / RETIRED）或已清理的批次不动。
+    """
+    state = client.eval(
+        _REJECT_LUA, 1, pc.versions_key(prefix), member, int(time.time()), (reason or "")[:200]
+    )
+    state = state.decode() if isinstance(state, bytes) else state
+    if state not in _DISCARDABLE_STATES:
+        logger.warning("skip rejecting batch %r on %r: state=%r", member, prefix, state)
+        return False
+    purge_batch(client, prefix, member)
+    logger.info("rejected and purged batch %r on %r: %s", member, prefix, reason)
+    return True
+
+
+# ---------------------------------------------------------------------------
+# 写入心跳（长时间同步不被判超时）
+# ---------------------------------------------------------------------------
+
+#: 心跳间隔，远小于 MAX_RUN_HOURS：Driver 卡顿几次也不至于被判超时
+HEARTBEAT_INTERVAL_SECONDS = 300
+
+#: 心跳最长持续时间：超过即停止续期，交给超时清理，防止失控的 Driver 永久占住门禁
+HEARTBEAT_MAX_SECONDS = 24 * 3600
+
+#: 只给仍是 WRITING 的批次刷新 updated_at，返回 1 / 0
+_HEARTBEAT_LUA = r"""
+local raw = redis.call('HGET', KEYS[1], ARGV[1])
+if not raw then
+  return 0
+end
+local info = cjson.decode(raw)
+if info.state ~= 'WRITING' then
+  return 0
+end
+info.updated_at = tonumber(ARGV[2])
+redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(info))
+return 1
+"""
+
+#: 仍是 WRITING 且 updated_at 与判超时时看到的一致（期间没被激活、没续心跳）才改为 ABANDONED，返回 1 / 0
+_MARK_ABANDONED_LUA = r"""
+local raw = redis.call('HGET', KEYS[1], ARGV[1])
+if not raw then
+  return 0
+end
+local info = cjson.decode(raw)
+if info.state ~= 'WRITING' or (tonumber(info.updated_at) or 0) ~= tonumber(ARGV[2]) then
+  return 0
+end
+info.state = 'ABANDONED'
+info.updated_at = tonumber(ARGV[3])
+redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(info))
+return 1
+"""
+
+
+class BatchHeartbeat:
+    """
+    写入期间在后台线程里定期刷新批次 ``updated_at``。
+
+    不刷新的话，超过 ``MAX_RUN_HOURS`` 的同步会被下一次运行的门禁判为 ABANDONED 并删掉数据，
+    而本次运行随后仍可能激活它，``__active__`` 指向已删除的批次，读侧全量 miss。
+    批次一旦不再是 WRITING（已激活 / 被拒绝 / 被判超时），心跳自动结束。
+    """
+
+    def __init__(
+        self,
+        client: redis.Redis,
+        prefix: str,
+        member: str,
+        interval_seconds: float = HEARTBEAT_INTERVAL_SECONDS,
+        max_seconds: float = HEARTBEAT_MAX_SECONDS,
+    ) -> None:
+        self._client = client
+        self._prefix = prefix
+        self._member = member
+        self._interval = interval_seconds
+        self._max_seconds = max_seconds
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, name=f"batch-heartbeat-{member}", daemon=True)
+
+    def start(self) -> "BatchHeartbeat":
+        self._thread.start()
+        return self
+
+    def beat(self) -> bool:
+        """刷新一次，返回批次是否仍在写入中。"""
+        return bool(
+            self._client.eval(_HEARTBEAT_LUA, 1, pc.versions_key(self._prefix), self._member, int(time.time()))
+        )
+
+    def stop(self) -> None:
+        """停止心跳（幂等）。"""
+        self._stop.set()
+        if self._thread.is_alive() and self._thread is not threading.current_thread():
+            self._thread.join(timeout=5)
+
+    def _run(self) -> None:
+        deadline = time.monotonic() + self._max_seconds
+        while not self._stop.wait(self._interval):
+            if time.monotonic() > deadline:
+                logger.warning("heartbeat of batch %r on %r exceeded %ss, stop", self._member, self._prefix,
+                               self._max_seconds)
+                return
+            try:
+                if not self.beat():
+                    return
+            except Exception as exc:  # 单次失败不退出：下一轮再试，超时判定有 MAX_RUN_HOURS 的余量
+                logger.warning("heartbeat of batch %r on %r failed: %s", self._member, self._prefix, exc)
+
+
+def start_heartbeat(client: redis.Redis, prefix: str, member: str, **kwargs: Any) -> BatchHeartbeat:
+    """登记批次后立即调用；激活或失败收尾前调用返回值的 ``stop()``。"""
+    return BatchHeartbeat(client, prefix, member, **kwargs).start()
+
+
 # ---------------------------------------------------------------------------
 # 整表删除（删除在线表的一次性清理作业 / SDK delete_table 共用）
 # ---------------------------------------------------------------------------
@@ -273,6 +419,16 @@ def _is_abandoned(info: Dict[str, Any], now: float) -> bool:
     return (now - updated) > MAX_RUN_HOURS * 3600
 
 
+def _mark_abandoned(client: redis.Redis, prefix: str, member: str, seen: Dict[str, Any], now: float) -> bool:
+    """超时的 WRITING → ABANDONED（原子比较 updated_at），返回是否标记成功。"""
+    return bool(
+        client.eval(
+            _MARK_ABANDONED_LUA, 1, pc.versions_key(prefix), member,
+            int(seen.get(pc.BATCH_UPDATED_AT, 0)), int(now),
+        )
+    )
+
+
 def _get_active_member(client: redis.Redis, prefix: str) -> Optional[str]:
     member = client.get(pc.active_key(prefix))
     if member is None:

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/wedata/feature_engineering/feast/infra/online_store/redis_packed.py

@@ -179,15 +179,61 @@ class WedataRedisPackedOnlineStore(OnlineStore):
     # ------------------------------------------------------------------
 
     @staticmethod
-    def serialize_entity_key(entity_key_proto: EntityKeyProto) -> str:
-        """EntityKeyProto → 实体键 ek（按 join_keys 顺序取值，转义后用 ``|`` 连接，codec 设计 §2.6.3）。"""
+    def serialize_entity_key(entity_key_proto: EntityKeyProto, order: Optional[Sequence[int]] = None) -> str:
+        """
+        EntityKeyProto → 实体键 ek（转义后用 ``|`` 连接，codec 设计 §2.6.3）。
+
+        :param order: 按 ``meta.entity_keys`` 顺序取值的下标（见 :meth:`_entity_value_order`）；
+            None 表示 join_keys 顺序与 meta 一致，按原顺序取值
+        """
         values = []
         for val in entity_key_proto.entity_values:
             if val.WhichOneof("val") is None:
                 raise pc.PackedCodecError("entity value not set (None)")
             values.append(_proto_value_to_py(val))
+        if order is not None:
+            values = [values[i] for i in order]
         return pc.build_entity_key_from_values(values)
 
+    @staticmethod
+    def _entity_value_order(
+        join_keys: Sequence[str], meta: SchemaMeta, table_name: str, prefix: str
+    ) -> Optional[List[int]]:
+        """
+        F8：join_keys 与建表时的 ``meta.entity_keys`` 比对。
+
+        集合相同、顺序不同（如请求写成 ``{"b": 1, "a": 2}``，feast 按请求键序构造 join_keys）是合法请求，
+        返回按 meta 顺序取值的下标；顺序一致或未带 join_keys 返回 None；集合不同（registry 改名 / 传错 FV）抛错，
+        否则会拼出错误的 ek 静默 100% miss。
+        """
+        if not join_keys:
+            return None
+        keys = tuple(join_keys)
+        expected = tuple(meta.entity_keys)
+        if keys == expected:
+            return None
+        if len(set(keys)) != len(keys) or set(keys) != set(expected):
+            raise pc.PackedCodecError(
+                f"entity join_keys {list(keys)} do not match online table "
+                f"{table_name} (meta entity_keys={list(expected)}, prefix={prefix!r}); "
+                f"the feature view / registry has drifted from the published table"
+            )
+        position = {k: i for i, k in enumerate(keys)}
+        return [position[k] for k in expected]
+
+    def _serialize_entity_keys(
+        self, entity_keys: Sequence[EntityKeyProto], meta: SchemaMeta, table_name: str, prefix: str
+    ) -> List[str]:
+        """按 meta 顺序批量构造 ek；同一批里 join_keys 顺序相同的只比对一次。"""
+        orders: Dict[Tuple[str, ...], Optional[List[int]]] = {}
+        out = []
+        for ek in entity_keys:
+            join_keys = tuple(ek.join_keys)
+            if join_keys not in orders:
+                orders[join_keys] = self._entity_value_order(join_keys, meta, table_name, prefix)
+            out.append(self.serialize_entity_key(ek, orders[join_keys]))
+        return out
+
     # ------------------------------------------------------------------
     # 读路径（稳态 1 次 MGET）
     # ------------------------------------------------------------------
@@ -259,16 +305,8 @@ class WedataRedisPackedOnlineStore(OnlineStore):
         sv, dv = pc.parse_active_pointer(member)
         meta = self._get_meta(client, prefix, sv)
 
-        # F8：entity join_keys 与建表时的 meta.entity_keys 不一致（registry 重命名/重排/传错 FV）
-        # → 抛错。否则按错误顺序拼 ek 会静默 100% miss。
-        if entity_keys[0].join_keys:
-            read_join_keys = tuple(entity_keys[0].join_keys)
-            if read_join_keys != tuple(meta.entity_keys):
-                raise pc.PackedCodecError(
-                    f"entity join_keys {list(read_join_keys)} do not match online table "
-                    f"{table.name} (meta entity_keys={list(meta.entity_keys)}, prefix={prefix!r}); "
-                    f"the feature view / registry has drifted from the published table"
-                )
+        # F8：join_keys 与 meta.entity_keys 集合不同则抛错，顺序不同按 meta 顺序重排（见 _entity_value_order）
+        eks = self._serialize_entity_keys(entity_keys, meta, table.name, prefix)
 
         # 校验 requested_features ⊆ meta fields（防拼写错误静默丢列）
         if requested_features:
@@ -278,7 +316,7 @@ class WedataRedisPackedOnlineStore(OnlineStore):
                     f"requested features not in online table {table.name}: {sorted(unknown)}"
                 )
 
-        keys = [pc.data_key(prefix, sv, dv, self.serialize_entity_key(ek)) for ek in entity_keys]
+        keys = [pc.data_key(prefix, sv, dv, ek) for ek in eks]
         blobs = client.mget(keys)
 
         result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = []
@@ -326,6 +364,8 @@ class WedataRedisPackedOnlineStore(OnlineStore):
         meta = pc.build_meta_from_feature_view(table, source_table=_source_table(table))
         ttl = int(getattr(config.online_store, "ttl_seconds", 86400 * 30))
         max_bytes = int(getattr(config.online_store, "max_string_bytes", pc.DEFAULT_MAX_STRING_BYTES))
+        # 与读路径同一规则按 meta 顺序构造 ek；放在登记批次之前，主键非法时不留下 WRITING 批次
+        eks = self._serialize_entity_keys([d[0] for d in data], meta, table.name, prefix)
 
         member = self._get_active(client, prefix)
         with self._cache_lock:  # 写路径直接失效读缓存，保证写后读一致
@@ -350,14 +390,14 @@ class WedataRedisPackedOnlineStore(OnlineStore):
             member = pc.build_batch_member(sv, dv)
 
         pipe = client.pipeline(transaction=False)
-        for entity_key_proto, feature_map, event_ts, _created_ts in data:
+        for ek, (_entity_key_proto, feature_map, event_ts, _created_ts) in zip(eks, data):
             values = {
                 name: _proto_value_to_py(feature_map[name])
                 for name in meta.field_names
                 if name in feature_map
             }
             pc.validate_string_lengths(values, max_bytes)
-            key = pc.data_key(prefix, sv, dv, self.serialize_entity_key(entity_key_proto))
+            key = pc.data_key(prefix, sv, dv, ek)
             ts_micros = None if event_ts is None else event_time_to_micros(event_ts)
             pipe.set(key, pc.encode_row(meta, ts_micros, values), ex=ttl)
         pipe.execute()

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/wedata/feature_engineering/table_client/table_client.py

@@ -813,7 +813,7 @@ class FeatureEngineeringTableClient(AbstractBaseTableClient):
                 Properties 中没有 connectionId，则降级走 TCCatalog 模式。常见取值如 `DataLakeCatalog`，
                 具体需结合所在工作空间已配置的在线 catalog 决定，禁止由 SDK 默认兜底。Redis 不传
             trigger: [可选]周期性任务的调度配置
-            online_store_type: [可选]在线存储类型，"TENCENT_POSTGRESQL"（默认，向后兼容）或 "REDIS"。
+            online_store_type: [可选]在线存储类型，"TENCENT_POSTGRE"（默认）或 "REDIS"。
                 传 "REDIS" 时：在线表由离线表与 Redis 数据源确定，不需要在线库表名（传了即报错）；
                 不支持持续同步（trigger 的 TriggerMode=CONTINUE_RUN 即报错），只支持 SNAPSHOT / TIME_TRIGGER；
                 同一离线表只能发布到 PG 或 Redis 其中一种（服务端校验）；
@@ -830,11 +830,11 @@ class FeatureEngineeringTableClient(AbstractBaseTableClient):
         """
         # store 类型分派（不传 = PG，行为与历史版本完全一致）
         if online_store_type is None:
-            online_store_type = "TENCENT_POSTGRESQL"
+            online_store_type = "TENCENT_POSTGRE"
         online_store_type = online_store_type.upper()
-        if online_store_type not in ("TENCENT_POSTGRESQL", "REDIS"):
+        if online_store_type not in ("TENCENT_POSTGRE", "REDIS"):
             raise ValueError(
-                f"online_store_type must be 'TENCENT_POSTGRESQL' or 'REDIS', got {online_store_type!r}"
+                f"online_store_type must be 'TENCENT_POSTGRE' or 'REDIS', got {online_store_type!r}"
             )
         online_names = {
             "online_db_name": online_db_name,
@@ -868,12 +868,12 @@ class FeatureEngineeringTableClient(AbstractBaseTableClient):
             ]
             if given_redis:
                 raise ValueError(
-                    f"{', '.join(given_redis)} must not be set for TENCENT_POSTGRESQL: "
+                    f"{', '.join(given_redis)} must not be set for TENCENT_POSTGRE: "
                     f"the PG connection is resolved from online_catalog_name on the server side"
                 )
             missing = [k for k, v in online_names.items() if not v]
             if missing:
-                raise ValueError(f"{', '.join(missing)} is required for TENCENT_POSTGRESQL online table")
+                raise ValueError(f"{', '.join(missing)} is required for TENCENT_POSTGRE online table")
 
         # 构建完整表名
         full_table_name = common_utils.build_full_table_name(table_name, schema_name, catalog_name, self._spark)
@@ -1037,16 +1037,16 @@ class FeatureEngineeringTableClient(AbstractBaseTableClient):
             catalog_name: 数据目录名称
             schema_name: 数据库名称
             table_name: 表名
-            online_store_type: [可选]在线存储类型，"TENCENT_POSTGRESQL" 或 "REDIS"。不传时由服务端按标签判定；
+            online_store_type: [可选]在线存储类型，"TENCENT_POSTGRE" 或 "REDIS"。不传时由服务端按标签判定；
                 传了也以标签为准（不一致时服务端告警）。
         Raises:
             RuntimeError: 当删除操作失败时抛出
         """
         if online_store_type is not None:
             online_store_type = online_store_type.upper()
-            if online_store_type not in ("TENCENT_POSTGRESQL", "REDIS"):
+            if online_store_type not in ("TENCENT_POSTGRE", "REDIS"):
                 raise ValueError(
-                    f"online_store_type must be 'TENCENT_POSTGRESQL' or 'REDIS', got {online_store_type!r}"
+                    f"online_store_type must be 'TENCENT_POSTGRE' or 'REDIS', got {online_store_type!r}"
                 )
 
         # 接口分页参数

{wedata3_feature_engineering-0.2.6.dev2 → wedata3-feature-engineering-0.2.6.dev4}/wedata3_feature_engineering.egg-info/PKG-INFO

@@ -1,41 +1,20 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: wedata3-feature-engineering
-Version: 0.2.6.dev2
+Version: 0.2.6.dev4
 Summary: Wedata Feature Engineering Library Development
-Home-page: 
+Home-page: UNKNOWN
 Author: meahqian
 Author-email: 
 License: Apache 2.0
+Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-Requires-Dist: pandas>=1.0.0
-Requires-Dist: feast[postgres,redis]==0.49.0
-Requires-Dist: grpcio==1.74.0
-Requires-Dist: msgpack<2,>=1.0
-Requires-Dist: tencentcloud-sdk-python
-Requires-Dist: ipython
-Requires-Dist: msgpack<2.0.0,>=1.0.0
 Provides-Extra: mlflow2
-Requires-Dist: mlflow==2.17.2; extra == "mlflow2"
 Provides-Extra: mlflow3
-Requires-Dist: mlflow<3.11.0,>=3.10.0; extra == "mlflow3"
 Provides-Extra: dev
-Requires-Dist: flake8; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: python-dotenv; extra == "dev"
-Requires-Dist: fakeredis[lua]; extra == "dev"
-Dynamic: author
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: license
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
 
 # wedata3-feature-engineering
 
@@ -160,3 +139,5 @@ bash build.sh
 ## 许可证
 
 Apache License 2.0
+
+