wedata3-feature-engineering 0.2.6.dev2__tar.gz → 0.2.6.dev3__tar.gz

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wedata3-feature-engineering
-Version: 0.2.6.dev2
+Version: 0.2.6.dev3
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

wedata3_feature_engineering-0.2.6.dev3/tests/test_build_executor_deps.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""
+scripts/build_executor_deps.sh 的静态检查（实跑需要 Linux x86_64 + 网络，放在发布流程里做）。
+
+曾经的三个问题，任何一个都会让打出来的包在 Executor 上不可用：
+1. 包名写成 tencent-wedata-feature-engineering，与 setup.py 不一致，pip 装不到；
+2. --python-version 39 与 python_requires>=3.10 冲突，pip 拒装；
+3. tar 打了外层 deps 目录，按 #deps 解压后变成 ./deps/deps/wedata，PYTHONPATH=./deps 找不到。
+"""
+
+import re
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = (ROOT / "scripts" / "build_executor_deps.sh").read_text(encoding="utf-8")
+
+
+def _setup_value(key: str) -> str:
+    match = re.search(rf"{key}\s*=\s*['\"]([^'\"]+)['\"]", (ROOT / "setup.py").read_text(encoding="utf-8"))
+    assert match, f"setup.py 中找不到 {key}"
+    return match.group(1)
+
+
+def test_package_name_matches_setup_py():
+    assert f'SDK_PACKAGE="{_setup_value("name")}"' in SCRIPT
+    assert "tencent-wedata-feature-engineering" not in SCRIPT
+
+
+def test_default_sdk_version_matches_constants():
+    from wedata.common.constants.constants import FEATURE_LOOKUP_CLIENT_MAJOR_VERSION
+
+    assert f'SDK_VERSION="${{SDK_VERSION:-{FEATURE_LOOKUP_CLIENT_MAJOR_VERSION}}}"' in SCRIPT
+
+
+def test_target_python_satisfies_python_requires():
+    required = _setup_value("python_requires")
+    minimum = tuple(int(x) for x in required.lstrip(">=").split("."))
+    default = re.search(r'PY_VERSION="\$\{PY_VERSION:-([\d.]+)\}"', SCRIPT).group(1)
+    assert tuple(int(x) for x in default.split(".")) >= minimum
+    assert '--python-version "$PY_VERSION"' in SCRIPT
+
+
+def test_archive_root_is_package_dir():
+    """归档里不能再套一层 deps：Spark 按 #deps 解压，PYTHONPATH=./deps 下要直接看到 wedata / msgpack。"""
+    assert 'tar -czf "${OUT_DIR}/${ARCHIVE_NAME}" -C "$STAGE_DIR/deps" .' in SCRIPT
+    assert not re.search(r'-C "\$STAGE_DIR" deps', SCRIPT)
+
+
+def test_no_invalid_spark_conf_and_no_shasum():
+    code = "\n".join(line for line in SCRIPT.splitlines() if not line.lstrip().startswith("#"))
+    assert "spark.driverEnv" not in code  # 不是 Spark 配置项
+    assert not re.search(r"\b(shasum|sha256sum)\s", code)  # slim 镜像不一定有

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/tests/test_cleanup_packed.py

@@ -101,6 +101,146 @@ class TestPrecheck:
         assert not client.exists(pc.data_key(PREFIX, sv(1), dv(1), "u1"))
 
 
+# ---------------------------------------------------------------------------
+# 作业失败收尾
+# ---------------------------------------------------------------------------
+
+def _state(client, member, prefix=PREFIX):
+    return pc.decode_batch_info(client.hget(pc.versions_key(prefix), member))[pc.BATCH_STATE]
+
+
+class TestRejectBatch:
+    def test_writing_rejected_and_purged_so_next_sync_is_not_blocked(self):
+        """作业失败后立即收尾：下一次同步的门禁不用等 MAX_RUN_HOURS。"""
+        client = make_client()
+        member = register(client, PREFIX, sv(1), dv(1), pc.STATE_WRITING)
+
+        assert cp.reject_batch(client, PREFIX, member, "row count mismatch") is True
+
+        assert _state(client, member) == pc.STATE_PURGED
+        assert not client.exists(pc.data_key(PREFIX, sv(1), dv(1), "u1"))
+        assert pc.decode_batch_info(client.hget(pc.versions_key(PREFIX), member))["reason"] == "row count mismatch"
+        cp.precheck_and_purge(client, PREFIX)
+
+    def test_stale_batch_purged_immediately(self):
+        client = make_client()
+        member = register(client, PREFIX, sv(1), dv(1), pc.STATE_STALE)
+
+        assert cp.reject_batch(client, PREFIX, member, "activate refused: STALE") is True
+        assert _state(client, member) == pc.STATE_PURGED
+        assert not client.exists(pc.data_key(PREFIX, sv(1), dv(1), "u2"))
+
+    @pytest.mark.parametrize("state", [pc.STATE_ACTIVE, pc.STATE_RETIRED, pc.STATE_PURGED])
+    def test_activated_or_finished_batch_untouched(self, state):
+        """激活脚本已执行、只是客户端没收到回包：批次已是 ACTIVE，绝不能被删。"""
+        client = make_client()
+        member = register(client, PREFIX, sv(1), dv(1), state)
+
+        assert cp.reject_batch(client, PREFIX, member, "activate failed") is False
+        assert _state(client, member) == state
+        assert client.exists(pc.data_key(PREFIX, sv(1), dv(1), "u1"))
+
+    def test_unknown_batch_is_noop(self):
+        client = make_client()
+        assert cp.reject_batch(client, PREFIX, pc.build_batch_member(sv(9), dv(9)), "x") is False
+        assert all_keys(client) == []
+
+
+# ---------------------------------------------------------------------------
+# 长时间同步：心跳 / 原子判超时 / 只激活 WRITING
+# ---------------------------------------------------------------------------
+
+def _activate(client, member, now=None):
+    sw, _prev, reason = client.eval(
+        pc.ACTIVATE_LUA, 2, pc.active_key(PREFIX), pc.versions_key(PREFIX), member,
+        int(now if now is not None else time.time()),
+    )
+    return int(sw), reason.decode() if isinstance(reason, bytes) else reason
+
+
+class TestLongRunningSync:
+    def test_heartbeat_keeps_long_run_from_being_abandoned(self):
+        """超过 MAX_RUN_HOURS 的同步只要心跳在续，就不会被另一次运行判超时删掉。"""
+        client = make_client()
+        stale = int(time.time()) - 7 * 3600
+        member = register(client, PREFIX, sv(1), dv(1), pc.STATE_WRITING, updated_at=stale)
+
+        hb = cp.BatchHeartbeat(client, PREFIX, member)
+        assert hb.beat() is True
+
+        with pytest.raises(cp.PrecheckError, match="in progress"):
+            cp.precheck_and_purge(client, PREFIX)
+        assert client.exists(pc.data_key(PREFIX, sv(1), dv(1), "u1"))
+
+    def test_heartbeat_thread_refreshes_and_stops(self):
+        client = make_client()
+        member = register(client, PREFIX, sv(1), dv(1), pc.STATE_WRITING, updated_at=1)
+
+        hb = cp.start_heartbeat(client, PREFIX, member, interval_seconds=0.01)
+        deadline = time.time() + 2
+        while time.time() < deadline and pc.decode_batch_info(
+                client.hget(pc.versions_key(PREFIX), member))[pc.BATCH_UPDATED_AT] == 1:
+            time.sleep(0.01)
+        hb.stop()
+
+        assert pc.decode_batch_info(client.hget(pc.versions_key(PREFIX), member))[pc.BATCH_UPDATED_AT] > 1
+        assert not hb._thread.is_alive()
+
+    def test_heartbeat_does_not_revive_finished_batch(self):
+        client = make_client()
+        member = register(client, PREFIX, sv(1), dv(1), pc.STATE_ACTIVE, updated_at=1)
+        assert cp.BatchHeartbeat(client, PREFIX, member).beat() is False
+        assert pc.decode_batch_info(client.hget(pc.versions_key(PREFIX), member))[pc.BATCH_UPDATED_AT] == 1
+
+    def test_abandoned_marking_is_atomic_against_activation(self):
+        """判超时之后、改状态之前批次被激活：不能再标 ABANDONED，更不能删数据。"""
+        client = make_client()
+        member = register(client, PREFIX, sv(1), dv(1), pc.STATE_WRITING, updated_at=int(time.time()) - 7 * 3600)
+        seen = pc.decode_batch_info(client.hget(pc.versions_key(PREFIX), member))
+        assert _activate(client, member)[0] == 1  # 慢作业恰好在此刻激活
+
+        assert cp._mark_abandoned(client, PREFIX, member, seen, time.time()) is False
+        assert cp.cleanup(client, PREFIX) == 0
+        assert _state(client, member) == pc.STATE_ACTIVE
+        assert client.exists(pc.data_key(PREFIX, sv(1), dv(1), "u1"))
+
+    def test_abandoned_batch_can_no_longer_be_activated(self):
+        """被判超时清理后，慢作业的激活必须被拒绝，__active__ 不能指向已删除的批次。"""
+        client = make_client()
+        cur = register(client, PREFIX, sv(1), dv(1), pc.STATE_WRITING)
+        assert _activate(client, cur)[0] == 1
+        slow = register(client, PREFIX, sv(1), dv(2), pc.STATE_WRITING, watermark=5000,
+                        updated_at=int(time.time()) - 7 * 3600)
+
+        cp.cleanup(client, PREFIX)
+        assert _state(client, slow) == pc.STATE_PURGED
+
+        assert _activate(client, slow) == (0, "NOT_WRITING")
+        assert client.get(pc.active_key(PREFIX)).decode() == cur
+
+
+class TestFutureWatermark:
+    def test_dirty_future_watermark_does_not_block_later_batches(self):
+        client = make_client()
+        now = int(time.time())
+        dirty = register(client, PREFIX, sv(1), dv(1), pc.STATE_WRITING, watermark=now + 30 * 86400)
+        assert _activate(client, dirty, now)[0] == 1
+        fixed = register(client, PREFIX, sv(1), dv(2), pc.STATE_WRITING, watermark=now - 60)
+
+        assert _activate(client, fixed, now)[0] == 1
+        assert client.get(pc.active_key(PREFIX)).decode() == fixed
+
+    def test_watermark_within_tolerance_still_compared(self):
+        """时区错配（8 小时内）的水位仍是有效水位，旧数据不能覆盖它。"""
+        client = make_client()
+        now = int(time.time())
+        cur = register(client, PREFIX, sv(1), dv(1), pc.STATE_WRITING, watermark=now + 8 * 3600)
+        assert _activate(client, cur, now)[0] == 1
+        older = register(client, PREFIX, sv(1), dv(2), pc.STATE_WRITING, watermark=now - 60)
+
+        assert _activate(client, older, now) == (0, "STALE")
+
+
 # ---------------------------------------------------------------------------
 # 状态规则
 # ---------------------------------------------------------------------------

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/tests/test_packed_codec.py

@@ -167,6 +167,51 @@ def test_length_tolerance():
     assert pc.decode_row(meta, long)[1] == {"age": 1, "city_id": 2, "ctr_7d": 3.0, "name": "n"}
 
 
+def test_decimal_and_date_columns_encode_by_declared_type():
+    """Spark DecimalType / DateType 的值是 Decimal / date；Java 把它们映射为 STRING，msgpack 不能直接序列化。"""
+    meta = make_meta(
+        fields=(
+            ("amount", "STRING"),
+            ("rate", "DOUBLE"),
+            ("cnt", "INT64"),
+            ("birthday", "STRING"),
+            ("signup", "UNIX_TIMESTAMP"),
+            ("seen_at", "STRING"),
+        )
+    )
+    values = {
+        "amount": Decimal("12.30"),
+        "rate": Decimal("0.125"),
+        "cnt": Decimal("42"),
+        "birthday": dt.date(1990, 5, 17),
+        "signup": dt.date(2026, 1, 2),
+        "seen_at": dt.datetime(2026, 1, 2, 3, 4, 5),
+    }
+    _, out = pc.decode_row(meta, pc.encode_row(meta, 1, values))
+    assert out == {
+        "amount": "12.30",
+        "rate": 0.125,
+        "cnt": 42,
+        "birthday": "1990-05-17",
+        "signup": int(dt.datetime(2026, 1, 2, tzinfo=dt.timezone.utc).timestamp()),
+        "seen_at": "2026-01-02T03:04:05",
+    }
+    # 不出现科学计数法；NaN Decimal 按缺失处理
+    assert pc.decode_row(meta, pc.encode_row(meta, 1, {"amount": Decimal("1E+2"), "rate": Decimal("NaN")}))[1] == {
+        "amount": "100"
+    }
+
+
+def test_unencodable_value_reports_column():
+    meta = make_meta(fields=(("cnt", "INT64"), ("flag", "BOOL"), ("tags", "STRING")))
+    with pytest.raises(PackedCodecError, match="cnt"):
+        pc.encode_row(meta, 1, {"cnt": Decimal("1.5")})
+    with pytest.raises(PackedCodecError, match="flag"):
+        pc.encode_row(meta, 1, {"flag": dt.date(2026, 1, 1)})
+    with pytest.raises(PackedCodecError, match="tags"):
+        pc.encode_row(meta, 1, {"tags": object()})
+
+
 def test_meta_roundtrip_and_codec_version_guard():
     meta = make_meta()
     assert SchemaMeta.from_bytes(meta.to_bytes()) == meta

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/tests/test_redis_packed.py

@@ -57,6 +57,12 @@ class FakeFVNoTag(FakeFV):
     tags = {}
 
 
+class FakeCompositeFV(FakeFV):
+    """复合主键：meta.entity_keys = (member_id, dept_id)"""
+
+    entity_columns = [_F("member_id", "STRING"), _F("dept_id", "STRING")]
+
+
 def make_store():
     store = WedataRedisPackedOnlineStore()
     store._client = fakeredis.FakeRedis()
@@ -219,6 +225,49 @@ class TestWriteReadRoundtrip:
         with pytest.raises(pc.PackedCodecError, match="join_keys"):
             store.online_read(config, fv, [wrong])
 
+    def test_composite_key_order_independent(self):
+        """F8 只拦集合不同：复合主键请求写成 {dept, member}（feast 按请求键序构造 join_keys）仍能读到。"""
+        store, config = make_store()
+        fv = FakeCompositeFV()
+        written = EntityKeyProto(
+            join_keys=["member_id", "dept_id"],
+            entity_values=[ValueProto(string_val="10086"), ValueProto(string_val="SH001")],
+        )
+        store.online_write_batch(config, fv, [(written, row(25, "北京", 3.5), ts(), None)])
+
+        reversed_order = EntityKeyProto(
+            join_keys=["dept_id", "member_id"],
+            entity_values=[ValueProto(string_val="SH001"), ValueProto(string_val="10086")],
+        )
+        [(_, values)] = store.online_read(config, fv, [reversed_order])
+        assert values["age"].int64_val == 25
+
+        partial = EntityKeyProto(join_keys=["member_id"], entity_values=[ValueProto(string_val="10086")])
+        with pytest.raises(pc.PackedCodecError, match="join_keys"):
+            store.online_read(config, fv, [partial])
+        duplicated = EntityKeyProto(
+            join_keys=["member_id", "member_id"],
+            entity_values=[ValueProto(string_val="10086"), ValueProto(string_val="10086")],
+        )
+        with pytest.raises(pc.PackedCodecError, match="join_keys"):
+            store.online_read(config, fv, [duplicated])
+
+    def test_composite_key_written_in_other_order_lands_on_same_key(self):
+        store, config = make_store()
+        fv = FakeCompositeFV()
+        reversed_order = EntityKeyProto(
+            join_keys=["dept_id", "member_id"],
+            entity_values=[ValueProto(string_val="SH001"), ValueProto(string_val="10086")],
+        )
+        store.online_write_batch(config, fv, [(reversed_order, row(30, "上海", 4.0), ts(), None)])
+
+        written = EntityKeyProto(
+            join_keys=["member_id", "dept_id"],
+            entity_values=[ValueProto(string_val="10086"), ValueProto(string_val="SH001")],
+        )
+        [(_, values)] = store.online_read(config, fv, [written])
+        assert values["age"].int64_val == 30
+
     def test_roundtrip(self):
         store, config = make_store()
         fv = FakeFV()

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/tests/test_sync_template.py

@@ -24,6 +24,11 @@ _TEMPLATE_RELPATH = os.path.join(
     "resources", "templates", "offline_to_online_3.0_redis.ftl",
 )
 
+_FEATURE_SERVICE_JAVA_RELPATH = os.path.join(
+    "..", "..", "..", "science", "server", "sci-feature-service", "src", "main", "java",
+    "com", "tencent", "cloud", "wedata", "feature", "service", "FeatureService.java",
+)
+
 #: Java genScriptContentRedis 必须提供的全部 FreeMarker 变量（两侧契约，改动需同步）
 EXPECTED_VARS = {
     "WORKSPACE_ID", "FEAST_SERVICE_IP", "FEAST_SERVICE_PORT",
@@ -42,7 +47,7 @@ _SAMPLE_VALUES = {
     "FEATURE_VIEW_NAME": "member_feat",
     "REDIS_CONNECTION_STRING": "redis://wedata:Wedata%402024@10.1.2.3:6379/0",
     "ONLINE_TABLE_PREFIX": "1001:k3j5h7g9d2s4",
-    "SDK_VERSION": "0.2.6.dev2",
+    "SDK_VERSION": "0.2.6.dev3",
     "PARTITIONS": "256",
     "TTL_SECONDS": "2592000",
     "VERIFY_SAMPLE_SIZE": "100",
@@ -139,6 +144,18 @@ class TestTemplateStructure:
         # 包名必须与 setup.py 一致（wedata3-feature-engineering），版本为 Java 注入
         assert "wedata3-feature-engineering==" in first
 
+    def test_java_default_sdk_version_matches_package(self):
+        """Java 渲染模板时 SDK_VERSION 的默认值必须是当前包版本：模板调用的 cleanup_packed.reject_batch 等只在新版本里有。"""
+        if _template_path() is None:
+            pytest.skip("template not found (science Java module not checked out)")
+        from wedata.common.constants.constants import FEATURE_LOOKUP_CLIENT_MAJOR_VERSION
+
+        java = os.path.abspath(os.path.join(os.path.dirname(__file__), _FEATURE_SERVICE_JAVA_RELPATH))
+        with open(java, encoding="utf-8") as f:
+            match = re.search(r"\$\{feature\.redis-sync\.sdk-version:([^}]+)\}", f.read())
+        assert match, "FeatureService.java 里找不到 feature.redis-sync.sdk-version 默认值"
+        assert match.group(1) == FEATURE_LOOKUP_CLIENT_MAJOR_VERSION
+
 
 class TestTemplateCorrectnessContracts:
     """红线契约：模板必须复用单一事实来源，禁止手拼。"""
@@ -181,6 +198,35 @@ class TestTemplateCorrectnessContracts:
         assert "STATE_WRITING" in all_src, "batch must be registered before writing"
         assert "cleanup" in all_src
 
+    def test_failures_after_register_reject_the_batch(self):
+        """登记批次之后的每一步失败都要 reject_batch 收尾，否则批次停在 WRITING，下一次同步被锁 6 小时。"""
+        _, nb = _rendered_nb()
+        cells = {c.get("id"): "".join(c["source"]) for c in _code_cells(nb)}
+        assert "cp.reject_batch(redis_client, online_table_prefix, batch_member" in cells["redis-register-batch"]
+        for cell_id in ("redis-spark-write", "redis-verify", "redis-activate"):
+            assert "_fail_batch(" in cells[cell_id], f"{cell_id} must reject the batch on failure"
+        assert "left in WRITING" not in "\n".join(cells.values())
+
+    def test_heartbeat_covers_writing_phase(self):
+        """登记后开始心跳，失败收尾与激活前都要停：否则运行超过 6 小时会被判超时删数据，或心跳续住一个死批次。"""
+        _, nb = _rendered_nb()
+        cells = {c.get("id"): "".join(c["source"]) for c in _code_cells(nb)}
+        register = cells["redis-register-batch"]
+        assert register.index("redis_client.hset(") < register.index("cp.start_heartbeat(")
+        fail_batch = register[register.index("def _fail_batch"):]
+        assert fail_batch.index("_heartbeat.stop()") < fail_batch.index("cp.reject_batch(")
+        activate = cells["redis-activate"]
+        assert activate.index("_heartbeat.stop()") < activate.index("pc.ACTIVATE_LUA")
+
+    def test_dirty_data_guards(self):
+        """未来时间戳截断水位（否则之后所有批次都被判旧）；主键 NULL 先过滤（否则写到一半整批失败）。"""
+        _, nb = _rendered_nb()
+        cells = {c.get("id"): "".join(c["source"]) for c in _code_cells(nb)}
+        assert "pc.WATERMARK_FUTURE_TOLERANCE_SECONDS" in cells["redis-register-batch"]
+        load = cells["redis-load-dedupe"]
+        assert "isNull()" in load and "F.isnan(" in load
+        assert load.index("df.filter(~_null_key)") < load.index("expected_count = df.count()")
+
     def test_meta_from_fv_not_schema(self):
         _, nb = _rendered_nb()
         all_src = "\n".join("".join(c["source"]) for c in _code_cells(nb))

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/wedata/common/constants/constants.py

@@ -46,7 +46,7 @@ FEATURE_LOOKUP_CLIENT_PIP_PACKAGE = "wedata3-feature-engineering"
 # 特征查找版本号
 # FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.6.16"
 # FEATURE_LOOKUP_CLIENT_PIP_PACKAGE_VERSION = f"{FEATURE_LOOKUP_CLIENT_PIP_PACKAGE}>=0.6.16"
-FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.2.6.dev2"
+FEATURE_LOOKUP_CLIENT_MAJOR_VERSION = "0.2.6.dev3"
 FEATURE_LOOKUP_CLIENT_PIP_PACKAGE_VERSION = f"{FEATURE_LOOKUP_CLIENT_PIP_PACKAGE}>=0.2.4"
 
 # 特征存储内部数据目录

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/wedata/common/online_store/packed_codec.py

@@ -389,19 +389,58 @@ def _encode_value(vtype: str, v: Any) -> Any:
         return None
     if isinstance(v, float) and math.isnan(v):
         return None
-    if vtype == "UNIX_TIMESTAMP" and isinstance(v, _dt.datetime):
+    if isinstance(v, Decimal):
+        return _encode_decimal(vtype, v)
+    if isinstance(v, _dt.date):
+        return _encode_temporal(vtype, v)
+    return v
+
+
+def _encode_decimal(vtype: str, v: Decimal) -> Any:
+    """Spark DecimalType → Decimal。Java 侧把 decimal 映射为 STRING（未知类型降级），msgpack 不能直接序列化 Decimal。"""
+    if not v.is_finite():
+        return None
+    if vtype == "STRING":
+        return format(v, "f")  # 保留小数位、不出现科学计数法（Decimal('1E+2') → '100'）
+    if vtype in ("DOUBLE", "FLOAT"):
+        return float(v)
+    if vtype in ("INT32", "INT64"):
+        if v != v.to_integral_value():
+            raise PackedCodecError(f"小数 {v} 不能写入整数类型 {vtype}")
+        return int(v)
+    raise PackedCodecError(f"Decimal 不能写入 {vtype} 类型")
+
+
+def _encode_temporal(vtype: str, v: _dt.date) -> Any:
+    """datetime / date。UNIX_TIMESTAMP 存 epoch 秒；Java 把 date 映射为 STRING，按 ISO 格式存。"""
+    if vtype == "UNIX_TIMESTAMP":
+        if not isinstance(v, _dt.datetime):
+            v = _dt.datetime(v.year, v.month, v.day)  # date 按当天 0 点
         if v.tzinfo is None:
             v = v.replace(tzinfo=_dt.timezone.utc)
         return int(v.timestamp())
-    return v
+    if vtype == "STRING":
+        return v.isoformat()
+    raise PackedCodecError(f"{type(v).__name__} 不能写入 {vtype} 类型")
 
 
 def encode_row(meta: SchemaMeta, ts_micros: Optional[int], values: Mapping[str, Any]) -> bytes:
     """dict → 位置数组 msgpack；缺失字段编码为 nil。下标 0 固定为 event_ts_micros。"""
     arr: List[Any] = [None if ts_micros is None else int(ts_micros)]
     for name, vtype in meta.fields:
-        arr.append(_encode_value(vtype, values.get(name)))
-    return msgpack.packb(arr, use_bin_type=True)
+        try:
+            arr.append(_encode_value(vtype, values.get(name)))
+        except PackedCodecError as e:
+            raise PackedCodecError(f"特征 {name} 编码失败: {e}") from None
+    try:
+        return msgpack.packb(arr, use_bin_type=True)
+    except TypeError as e:
+        # 兜底：出现 codec 没覆盖的类型时报清楚是哪一列，而不是裸 msgpack 错误
+        bad = [n for (n, _), x in zip(meta.fields, arr[1:]) if not isinstance(x, _MSGPACK_SCALARS)]
+        raise PackedCodecError(f"特征 {bad} 的值类型不支持序列化: {e}") from None
+
+
+_MSGPACK_SCALARS = (type(None), bool, int, float, str, bytes, bytearray)
 
 
 def decode_row(
@@ -617,20 +656,35 @@ ACTIVATE_LUA = r"""
 -- 返回 {switched(1/0), previous_active or "", reason}
 local new_member = ARGV[1]
 local now = tonumber(ARGV[2])
+-- 水位超过 now + 1 天视为脏数据（未来时间戳），按 0 参与比较：
+-- 否则一条脏数据会让之后所有批次都被判旧（与 WATERMARK_FUTURE_TOLERANCE_SECONDS 一致）
+local MAX_FUTURE_SECONDS = 86400
+local function watermark_of(info)
+  if info.watermark > now + MAX_FUTURE_SECONDS then
+    return 0
+  end
+  return info.watermark
+end
 
 local new_raw = redis.call('HGET', KEYS[2], new_member)
 if not new_raw then
   return {0, '', 'NEW_NOT_REGISTERED'}
 end
 local new_info = cjson.decode(new_raw)
+-- 只激活写入中的批次：超时被判 ABANDONED、已 REJECTED / PURGED 的批次数据可能已被清理，激活即全量 miss
+if new_info.state ~= 'WRITING' then
+  return {0, redis.call('GET', KEYS[1]) or '', 'NOT_WRITING'}
+end
 
 local cur_member = redis.call('GET', KEYS[1])
 if cur_member then
   local cur_raw = redis.call('HGET', KEYS[2], cur_member)
   if cur_raw then
     local cur = cjson.decode(cur_raw)
-    local newer = (new_info.watermark > cur.watermark) or
-                  (new_info.watermark == cur.watermark and new_info.run_start > cur.run_start)
+    local new_wm = watermark_of(new_info)
+    local cur_wm = watermark_of(cur)
+    local newer = (new_wm > cur_wm) or
+                  (new_wm == cur_wm and new_info.run_start > cur.run_start)
     if not newer then
       new_info.state = 'STALE'
       new_info.updated_at = now
@@ -651,3 +705,7 @@ new_info.updated_at = now
 redis.call('HSET', KEYS[2], new_member, cjson.encode(new_info))
 return {1, cur_member or '', 'SWITCHED'}
 """
+
+#: 批次水位允许超出当前时间的上限（秒）。同步模板登记时超出即截断为当前时间并告警，ACTIVATE_LUA 比较时超出按 0 处理。
+#: 1 天能容纳时区错配（P1-13 的 8 小时偏移）与机器时钟偏差。
+WATERMARK_FUTURE_TOLERANCE_SECONDS = 86400

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/wedata/feature_engineering/feast/infra/online_store/cleanup_packed.py

@@ -15,6 +15,7 @@ key 格式与 BatchInfo 编解码统一走 ``wedata.common.online_store.packed_c
 | 状态 | 处置 |
 |---|---|
 | WRITING（未超时 <6h） | 不清理（同步进行中） |
+| WRITING（作业失败收尾）| ``reject_batch`` 置 REJECTED 并立即清理，不等 6h |
 | WRITING（超时 ≥6h）| 转 ABANDONED 后立即清理 |
 | ACTIVE | **永不清理**（二次确认防误删） |
 | RETIRED（最新 1 个，retired <1h） | 保留（回滚窗口） |
@@ -30,6 +31,7 @@ from __future__ import annotations
 
 import logging
 import re
+import threading
 import time
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -134,14 +136,15 @@ def cleanup(client: redis.Redis, prefix: str) -> int:
         if state in (pc.STATE_REJECTED, pc.STATE_STALE, pc.STATE_ABANDONED):
             to_delete.append((member, info))
         elif state == pc.STATE_WRITING and _is_abandoned(info, now):
-            _mark_state(client, prefix, member, pc.STATE_ABANDONED, now)
-            to_delete.append((member, info))
+            # 原子判定：期间批次被激活或续了心跳就不动它（快照判超时、再非原子改状态会把刚激活的批次删掉）
+            if _mark_abandoned(client, prefix, member, info, now):
+                to_delete.append((member, info))
         # ACTIVE / 未超时 WRITING / PURGED：不删数据
 
     deleted = 0
     for member, _info in to_delete:
-        if member == active_member:
-            # 防御性检查：理论不可能（ACTIVE 不在 to_delete），双重保险
+        # active 以删除前实时读取为准，不用上面的快照：快照之后可能已切换
+        if member in (active_member, _get_active_member(client, prefix)):
             logger.warning("skip purge of active batch %r on %r", member, prefix)
             continue
         try:
@@ -170,6 +173,149 @@ def purge_batch(client: redis.Redis, prefix: str, member: str) -> None:
     _mark_state(client, prefix, member, pc.STATE_PURGED, time.time())
 
 
+#: 只把仍处于 WRITING 的批次改为 REJECTED，返回改写后的状态（批次不存在返回空串）。
+#: 原子判断：激活脚本已执行但客户端没收到回包时，批次可能已是 ACTIVE，绝不能被改写。
+_REJECT_LUA = r"""
+local raw = redis.call('HGET', KEYS[1], ARGV[1])
+if not raw then
+  return ''
+end
+local info = cjson.decode(raw)
+if info.state == 'WRITING' then
+  info.state = 'REJECTED'
+  info.updated_at = tonumber(ARGV[2])
+  info.reason = ARGV[3]
+  redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(info))
+end
+return info.state
+"""
+
+#: 失败收尾时可以立即删除数据的状态（REJECTED 来自本函数，STALE 来自激活脚本判旧）
+_DISCARDABLE_STATES = frozenset({pc.STATE_REJECTED, pc.STATE_STALE})
+
+
+def reject_batch(client: redis.Redis, prefix: str, member: str, reason: str) -> bool:
+    """
+    同步作业失败收尾：本批 WRITING → REJECTED，并立即删除本批数据，返回是否删除了数据。
+
+    不收尾的话批次停在 WRITING，下一次同步的门禁要等 ``MAX_RUN_HOURS`` 才放行；
+    激活被判旧（STALE）的批次同样立即删除，不留一份全量数据占内存到下一次同步。
+    已激活（ACTIVE / RETIRED）或已清理的批次不动。
+    """
+    state = client.eval(
+        _REJECT_LUA, 1, pc.versions_key(prefix), member, int(time.time()), (reason or "")[:200]
+    )
+    state = state.decode() if isinstance(state, bytes) else state
+    if state not in _DISCARDABLE_STATES:
+        logger.warning("skip rejecting batch %r on %r: state=%r", member, prefix, state)
+        return False
+    purge_batch(client, prefix, member)
+    logger.info("rejected and purged batch %r on %r: %s", member, prefix, reason)
+    return True
+
+
+# ---------------------------------------------------------------------------
+# 写入心跳（长时间同步不被判超时）
+# ---------------------------------------------------------------------------
+
+#: 心跳间隔，远小于 MAX_RUN_HOURS：Driver 卡顿几次也不至于被判超时
+HEARTBEAT_INTERVAL_SECONDS = 300
+
+#: 心跳最长持续时间：超过即停止续期，交给超时清理，防止失控的 Driver 永久占住门禁
+HEARTBEAT_MAX_SECONDS = 24 * 3600
+
+#: 只给仍是 WRITING 的批次刷新 updated_at，返回 1 / 0
+_HEARTBEAT_LUA = r"""
+local raw = redis.call('HGET', KEYS[1], ARGV[1])
+if not raw then
+  return 0
+end
+local info = cjson.decode(raw)
+if info.state ~= 'WRITING' then
+  return 0
+end
+info.updated_at = tonumber(ARGV[2])
+redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(info))
+return 1
+"""
+
+#: 仍是 WRITING 且 updated_at 与判超时时看到的一致（期间没被激活、没续心跳）才改为 ABANDONED，返回 1 / 0
+_MARK_ABANDONED_LUA = r"""
+local raw = redis.call('HGET', KEYS[1], ARGV[1])
+if not raw then
+  return 0
+end
+local info = cjson.decode(raw)
+if info.state ~= 'WRITING' or (tonumber(info.updated_at) or 0) ~= tonumber(ARGV[2]) then
+  return 0
+end
+info.state = 'ABANDONED'
+info.updated_at = tonumber(ARGV[3])
+redis.call('HSET', KEYS[1], ARGV[1], cjson.encode(info))
+return 1
+"""
+
+
+class BatchHeartbeat:
+    """
+    写入期间在后台线程里定期刷新批次 ``updated_at``。
+
+    不刷新的话，超过 ``MAX_RUN_HOURS`` 的同步会被下一次运行的门禁判为 ABANDONED 并删掉数据，
+    而本次运行随后仍可能激活它，``__active__`` 指向已删除的批次，读侧全量 miss。
+    批次一旦不再是 WRITING（已激活 / 被拒绝 / 被判超时），心跳自动结束。
+    """
+
+    def __init__(
+        self,
+        client: redis.Redis,
+        prefix: str,
+        member: str,
+        interval_seconds: float = HEARTBEAT_INTERVAL_SECONDS,
+        max_seconds: float = HEARTBEAT_MAX_SECONDS,
+    ) -> None:
+        self._client = client
+        self._prefix = prefix
+        self._member = member
+        self._interval = interval_seconds
+        self._max_seconds = max_seconds
+        self._stop = threading.Event()
+        self._thread = threading.Thread(target=self._run, name=f"batch-heartbeat-{member}", daemon=True)
+
+    def start(self) -> "BatchHeartbeat":
+        self._thread.start()
+        return self
+
+    def beat(self) -> bool:
+        """刷新一次，返回批次是否仍在写入中。"""
+        return bool(
+            self._client.eval(_HEARTBEAT_LUA, 1, pc.versions_key(self._prefix), self._member, int(time.time()))
+        )
+
+    def stop(self) -> None:
+        """停止心跳（幂等）。"""
+        self._stop.set()
+        if self._thread.is_alive() and self._thread is not threading.current_thread():
+            self._thread.join(timeout=5)
+
+    def _run(self) -> None:
+        deadline = time.monotonic() + self._max_seconds
+        while not self._stop.wait(self._interval):
+            if time.monotonic() > deadline:
+                logger.warning("heartbeat of batch %r on %r exceeded %ss, stop", self._member, self._prefix,
+                               self._max_seconds)
+                return
+            try:
+                if not self.beat():
+                    return
+            except Exception as exc:  # 单次失败不退出：下一轮再试，超时判定有 MAX_RUN_HOURS 的余量
+                logger.warning("heartbeat of batch %r on %r failed: %s", self._member, self._prefix, exc)
+
+
+def start_heartbeat(client: redis.Redis, prefix: str, member: str, **kwargs: Any) -> BatchHeartbeat:
+    """登记批次后立即调用；激活或失败收尾前调用返回值的 ``stop()``。"""
+    return BatchHeartbeat(client, prefix, member, **kwargs).start()
+
+
 # ---------------------------------------------------------------------------
 # 整表删除（删除在线表的一次性清理作业 / SDK delete_table 共用）
 # ---------------------------------------------------------------------------
@@ -273,6 +419,16 @@ def _is_abandoned(info: Dict[str, Any], now: float) -> bool:
     return (now - updated) > MAX_RUN_HOURS * 3600
 
 
+def _mark_abandoned(client: redis.Redis, prefix: str, member: str, seen: Dict[str, Any], now: float) -> bool:
+    """超时的 WRITING → ABANDONED（原子比较 updated_at），返回是否标记成功。"""
+    return bool(
+        client.eval(
+            _MARK_ABANDONED_LUA, 1, pc.versions_key(prefix), member,
+            int(seen.get(pc.BATCH_UPDATED_AT, 0)), int(now),
+        )
+    )
+
+
 def _get_active_member(client: redis.Redis, prefix: str) -> Optional[str]:
     member = client.get(pc.active_key(prefix))
     if member is None:

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/wedata/feature_engineering/feast/infra/online_store/redis_packed.py

@@ -179,15 +179,61 @@ class WedataRedisPackedOnlineStore(OnlineStore):
     # ------------------------------------------------------------------
 
     @staticmethod
-    def serialize_entity_key(entity_key_proto: EntityKeyProto) -> str:
-        """EntityKeyProto → 实体键 ek（按 join_keys 顺序取值，转义后用 ``|`` 连接，codec 设计 §2.6.3）。"""
+    def serialize_entity_key(entity_key_proto: EntityKeyProto, order: Optional[Sequence[int]] = None) -> str:
+        """
+        EntityKeyProto → 实体键 ek（转义后用 ``|`` 连接，codec 设计 §2.6.3）。
+
+        :param order: 按 ``meta.entity_keys`` 顺序取值的下标（见 :meth:`_entity_value_order`）；
+            None 表示 join_keys 顺序与 meta 一致，按原顺序取值
+        """
         values = []
         for val in entity_key_proto.entity_values:
             if val.WhichOneof("val") is None:
                 raise pc.PackedCodecError("entity value not set (None)")
             values.append(_proto_value_to_py(val))
+        if order is not None:
+            values = [values[i] for i in order]
         return pc.build_entity_key_from_values(values)
 
+    @staticmethod
+    def _entity_value_order(
+        join_keys: Sequence[str], meta: SchemaMeta, table_name: str, prefix: str
+    ) -> Optional[List[int]]:
+        """
+        F8：join_keys 与建表时的 ``meta.entity_keys`` 比对。
+
+        集合相同、顺序不同（如请求写成 ``{"b": 1, "a": 2}``，feast 按请求键序构造 join_keys）是合法请求，
+        返回按 meta 顺序取值的下标；顺序一致或未带 join_keys 返回 None；集合不同（registry 改名 / 传错 FV）抛错，
+        否则会拼出错误的 ek 静默 100% miss。
+        """
+        if not join_keys:
+            return None
+        keys = tuple(join_keys)
+        expected = tuple(meta.entity_keys)
+        if keys == expected:
+            return None
+        if len(set(keys)) != len(keys) or set(keys) != set(expected):
+            raise pc.PackedCodecError(
+                f"entity join_keys {list(keys)} do not match online table "
+                f"{table_name} (meta entity_keys={list(expected)}, prefix={prefix!r}); "
+                f"the feature view / registry has drifted from the published table"
+            )
+        position = {k: i for i, k in enumerate(keys)}
+        return [position[k] for k in expected]
+
+    def _serialize_entity_keys(
+        self, entity_keys: Sequence[EntityKeyProto], meta: SchemaMeta, table_name: str, prefix: str
+    ) -> List[str]:
+        """按 meta 顺序批量构造 ek；同一批里 join_keys 顺序相同的只比对一次。"""
+        orders: Dict[Tuple[str, ...], Optional[List[int]]] = {}
+        out = []
+        for ek in entity_keys:
+            join_keys = tuple(ek.join_keys)
+            if join_keys not in orders:
+                orders[join_keys] = self._entity_value_order(join_keys, meta, table_name, prefix)
+            out.append(self.serialize_entity_key(ek, orders[join_keys]))
+        return out
+
     # ------------------------------------------------------------------
     # 读路径（稳态 1 次 MGET）
     # ------------------------------------------------------------------
@@ -259,16 +305,8 @@ class WedataRedisPackedOnlineStore(OnlineStore):
         sv, dv = pc.parse_active_pointer(member)
         meta = self._get_meta(client, prefix, sv)
 
-        # F8：entity join_keys 与建表时的 meta.entity_keys 不一致（registry 重命名/重排/传错 FV）
-        # → 抛错。否则按错误顺序拼 ek 会静默 100% miss。
-        if entity_keys[0].join_keys:
-            read_join_keys = tuple(entity_keys[0].join_keys)
-            if read_join_keys != tuple(meta.entity_keys):
-                raise pc.PackedCodecError(
-                    f"entity join_keys {list(read_join_keys)} do not match online table "
-                    f"{table.name} (meta entity_keys={list(meta.entity_keys)}, prefix={prefix!r}); "
-                    f"the feature view / registry has drifted from the published table"
-                )
+        # F8：join_keys 与 meta.entity_keys 集合不同则抛错，顺序不同按 meta 顺序重排（见 _entity_value_order）
+        eks = self._serialize_entity_keys(entity_keys, meta, table.name, prefix)
 
         # 校验 requested_features ⊆ meta fields（防拼写错误静默丢列）
         if requested_features:
@@ -278,7 +316,7 @@ class WedataRedisPackedOnlineStore(OnlineStore):
                     f"requested features not in online table {table.name}: {sorted(unknown)}"
                 )
 
-        keys = [pc.data_key(prefix, sv, dv, self.serialize_entity_key(ek)) for ek in entity_keys]
+        keys = [pc.data_key(prefix, sv, dv, ek) for ek in eks]
         blobs = client.mget(keys)
 
         result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = []
@@ -326,6 +364,8 @@ class WedataRedisPackedOnlineStore(OnlineStore):
         meta = pc.build_meta_from_feature_view(table, source_table=_source_table(table))
         ttl = int(getattr(config.online_store, "ttl_seconds", 86400 * 30))
         max_bytes = int(getattr(config.online_store, "max_string_bytes", pc.DEFAULT_MAX_STRING_BYTES))
+        # 与读路径同一规则按 meta 顺序构造 ek；放在登记批次之前，主键非法时不留下 WRITING 批次
+        eks = self._serialize_entity_keys([d[0] for d in data], meta, table.name, prefix)
 
         member = self._get_active(client, prefix)
         with self._cache_lock:  # 写路径直接失效读缓存，保证写后读一致
@@ -350,14 +390,14 @@ class WedataRedisPackedOnlineStore(OnlineStore):
             member = pc.build_batch_member(sv, dv)
 
         pipe = client.pipeline(transaction=False)
-        for entity_key_proto, feature_map, event_ts, _created_ts in data:
+        for ek, (_entity_key_proto, feature_map, event_ts, _created_ts) in zip(eks, data):
             values = {
                 name: _proto_value_to_py(feature_map[name])
                 for name in meta.field_names
                 if name in feature_map
             }
             pc.validate_string_lengths(values, max_bytes)
-            key = pc.data_key(prefix, sv, dv, self.serialize_entity_key(entity_key_proto))
+            key = pc.data_key(prefix, sv, dv, ek)
             ts_micros = None if event_ts is None else event_time_to_micros(event_ts)
             pipe.set(key, pc.encode_row(meta, ts_micros, values), ex=ttl)
         pipe.execute()

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/wedata3_feature_engineering.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wedata3-feature-engineering
-Version: 0.2.6.dev2
+Version: 0.2.6.dev3
 Summary: Wedata Feature Engineering Library Development
 Home-page: 
 Author: meahqian

{wedata3_feature_engineering-0.2.6.dev2 → wedata3_feature_engineering-0.2.6.dev3}/wedata3_feature_engineering.egg-info/SOURCES.txt

@@ -1,5 +1,6 @@
 README.md
 setup.py
+tests/test_build_executor_deps.py
 tests/test_cleanup_packed.py
 tests/test_client.py
 tests/test_common_utils.py