rocksdb-dedup 0.1.0__py3-none-any.whl

dedup_client/__init__.py

@@ -0,0 +1,22 @@
+"""
+RocksDB 去重服务 Python 客户端库
+
+基于 RocksDB + 布隆过滤器的高并发持久化去重引擎的 Python 客户端。
+
+快速开始:
+    from dedup_client import DedupClient
+
+    with DedupClient("localhost:50051") as client:
+        result = client.check_and_set(
+            namespace="spider_news",
+            key=b"https://example.com/page/1",
+            expire_at_ms=1735689600000,
+        )
+        print(result)  # "inserted" / "duplicate" / "expired_and_replaced"
+"""
+
+from .client import DedupClient
+from .sharded import ShardedDedupClient, ConsistentHashRing
+
+__all__ = ["DedupClient", "ShardedDedupClient", "ConsistentHashRing"]
+__version__ = "0.2.0"

dedup_client/client.py

@@ -0,0 +1,307 @@
+"""
+RocksDB 去重服务 gRPC 客户端
+
+使用示例:
+    from dedup_client import DedupClient
+
+    client = DedupClient("localhost:50051")
+
+    # 检查并写入
+    result = client.check_and_set(
+        namespace="spider_news",
+        key=b"https://example.com/page/1",
+        expire_at_ms=1735689600000,
+        metadata=b"some metadata",
+    )
+    print(result)  # "inserted" / "duplicate" / "expired_and_replaced"
+
+    # 仅检查是否存在
+    exists = client.exists("spider_news", b"https://example.com/page/1")
+"""
+
+import time
+from typing import Optional, List, Tuple
+
+import grpc
+
+from . import dedup_pb2
+from . import dedup_pb2_grpc
+
+
+class DedupClient:
+    """去重服务 gRPC 客户端
+
+    提供基于 RocksDB + 布隆过滤器的高性能持久化去重能力，
+    支持多租户 namespace 隔离、原子检查写入、批量操作等。
+
+    Attributes:
+        addr: 服务端地址，如 "localhost:50051"
+        timeout: 默认超时时间（秒）
+    """
+
+    def __init__(self, addr: str, timeout: float = 5.0):
+        """
+        Args:
+            addr: 服务地址，如 "localhost:50051"
+            timeout: 默认超时时间（秒）
+        """
+        self.addr = addr
+        self.timeout = timeout
+        self.channel = grpc.insecure_channel(addr)
+        self.stub = dedup_pb2_grpc.DedupServiceStub(self.channel)
+
+    def close(self):
+        """关闭连接"""
+        self.channel.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    # -----------------------------------------------------------------------
+    # 核心 API
+    # -----------------------------------------------------------------------
+
+    def check_and_set(
+        self,
+        namespace: str,
+        key: bytes,
+        expire_at_ms: int,
+        metadata: Optional[bytes] = None,
+    ) -> str:
+        """
+        检查 key 是否存在，不存在则写入（原子操作）
+
+        Args:
+            namespace: 业务命名空间，不同业务之间数据隔离
+            key: 去重键（bytes，支持任意二进制 key）
+            expire_at_ms: 绝对过期时间（毫秒时间戳）
+            metadata: 可选附加元数据
+
+        Returns:
+            "inserted" - 新写入
+            "duplicate" - 已存在（未过期）
+            "expired_and_replaced" - 已过期，已替换
+        """
+        req = dedup_pb2.CheckAndSetRequest(
+            namespace=namespace,
+            key=key,
+            expire_at_ms=expire_at_ms,
+            metadata=metadata or b"",
+        )
+        resp = self.stub.CheckAndSet(req, timeout=self.timeout)
+        return self._result_to_str(resp.result)
+
+    def batch_check_and_set(
+        self,
+        items: List[Tuple[str, bytes, int, Optional[bytes]]],
+    ) -> List[str]:
+        """
+        批量检查并写入（减少 RPC 次数，高 QPS 场景推荐）
+
+        Args:
+            items: [(namespace, key, expire_at_ms, metadata), ...]
+
+        Returns:
+            结果列表，顺序与输入一致
+        """
+        req_items = []
+        for ns, key, expire_at, meta in items:
+            req_items.append(dedup_pb2.CheckAndSetRequest(
+                namespace=ns,
+                key=key,
+                expire_at_ms=expire_at,
+                metadata=meta or b"",
+            ))
+        req = dedup_pb2.BatchCheckAndSetRequest(items=req_items)
+        resp = self.stub.BatchCheckAndSet(req, timeout=self.timeout)
+        return [self._result_to_str(r.result) for r in resp.results]
+
+    def exists(self, namespace: str, key: bytes) -> bool:
+        """仅检查 key 是否存在且未过期
+
+        Args:
+            namespace: 业务命名空间
+            key: 去重键
+
+        Returns:
+            True 表示存在且未过期，False 表示不存在或已过期
+        """
+        req = dedup_pb2.ExistsRequest(namespace=namespace, key=key)
+        resp = self.stub.Exists(req, timeout=self.timeout)
+        return resp.exists
+
+    def batch_exists(self, namespace: str, keys: List[bytes]) -> List[bool]:
+        """批量检查是否存在
+
+        Args:
+            namespace: 业务命名空间（同一 namespace 下的多个 key）
+            keys: 去重键列表
+
+        Returns:
+            存在性列表，顺序与输入一致
+        """
+        req = dedup_pb2.BatchExistsRequest(namespace=namespace, keys=keys)
+        resp = self.stub.BatchExists(req, timeout=self.timeout)
+        return list(resp.exists)
+
+    def get(self, namespace: str, key: bytes) -> Optional[dict]:
+        """
+        获取 key 的详细信息
+
+        Args:
+            namespace: 业务命名空间
+            key: 去重键
+
+        Returns:
+            {
+                "expire_at_ms": int,
+                "created_at_ms": int,
+                "metadata": bytes,
+            }
+            不存在返回 None
+        """
+        req = dedup_pb2.GetRequest(namespace=namespace, key=key)
+        resp = self.stub.Get(req, timeout=self.timeout)
+        if not resp.found:
+            return None
+        return {
+            "expire_at_ms": resp.value.expire_at_ms,
+            "created_at_ms": resp.value.created_at_ms,
+            "metadata": resp.value.metadata,
+        }
+
+    def delete(self, namespace: str, key: bytes) -> bool:
+        """删除 key
+
+        Args:
+            namespace: 业务命名空间
+            key: 去重键
+
+        Returns:
+            True 表示成功删除，False 表示 key 不存在
+        """
+        req = dedup_pb2.DeleteRequest(namespace=namespace, key=key)
+        resp = self.stub.Delete(req, timeout=self.timeout)
+        return resp.deleted
+
+    def batch_delete(self, namespace: str, keys: List[bytes]) -> List[bool]:
+        """批量删除
+
+        Args:
+            namespace: 业务命名空间
+            keys: 去重键列表
+
+        Returns:
+            删除结果列表，顺序与输入一致
+        """
+        req = dedup_pb2.BatchDeleteRequest(namespace=namespace, keys=keys)
+        resp = self.stub.BatchDelete(req, timeout=self.timeout)
+        return list(resp.deleted)
+
+    def stats(self, namespace: str = "") -> dict:
+        """
+        获取服务统计信息
+
+        Args:
+            namespace: 可选，指定 namespace；留空返回全局统计
+
+        Returns:
+            {
+                "total_keys": int,
+                "bloom_capacity": int,
+                "bloom_insertions": int,
+                "bloom_fpp": float,
+                "cleaned_total": int,
+                "backlog_estimate": int,
+                "current_speed": int,
+                "qps_estimate": int,
+                "error_count": int,
+            }
+        """
+        req = dedup_pb2.StatsRequest(namespace=namespace)
+        resp = self.stub.Stats(req, timeout=self.timeout)
+        return {
+            "total_keys": resp.total_keys,
+            "bloom_capacity": resp.bloom_capacity,
+            "bloom_insertions": resp.bloom_insertions,
+            "bloom_fpp": resp.bloom_fpp,
+            "cleaned_total": resp.cleaned_total,
+            "backlog_estimate": resp.backlog_estimate,
+            "current_speed": resp.current_speed,
+            "qps_estimate": resp.qps_estimate,
+            "error_count": resp.error_count,
+        }
+
+    # -----------------------------------------------------------------------
+    # 工具方法
+    # -----------------------------------------------------------------------
+
+    @staticmethod
+    def _result_to_str(result: int) -> str:
+        """将 proto enum 转换为字符串"""
+        if result == dedup_pb2.CheckResult.INSERTED:
+            return "inserted"
+        elif result == dedup_pb2.CheckResult.DUPLICATE:
+            return "duplicate"
+        elif result == dedup_pb2.CheckResult.EXPIRED_AND_REPLACED:
+            return "expired_and_replaced"
+        else:
+            return f"unknown({result})"
+
+
+# ---------------------------------------------------------------------------
+#  使用示例
+# ---------------------------------------------------------------------------
+
+def main():
+    """简单的使用示例"""
+    print("连接去重服务...")
+    with DedupClient("localhost:50051") as client:
+        ns = "spider_news"
+        expire_at = int(time.time() * 1000) + 30 * 86400 * 1000  # 30天后
+
+        # 测试 check_and_set
+        print("\n--- 测试 check_and_set ---")
+        result = client.check_and_set(ns, b"https://example.com/1", expire_at)
+        print(f"第一次写入: {result}")  # inserted
+
+        result = client.check_and_set(ns, b"https://example.com/1", expire_at)
+        print(f"第二次写入: {result}")  # duplicate
+
+        # 测试 exists
+        print("\n--- 测试 exists ---")
+        print(f"存在: {client.exists(ns, b'https://example.com/1')}")  # True
+        print(f"不存在: {client.exists(ns, b'https://example.com/999')}")  # False
+
+        # 测试 get
+        print("\n--- 测试 get ---")
+        value = client.get(ns, b"https://example.com/1")
+        print(f"详情: {value}")
+
+        # 测试批量
+        print("\n--- 测试 batch_check_and_set ---")
+        items = [
+            (ns, f"https://example.com/batch_{i}".encode(), expire_at, None)
+            for i in range(10)
+        ]
+        results = client.batch_check_and_set(items)
+        print(f"批量结果: {results}")
+
+        # 测试 stats
+        print("\n--- 服务状态 ---")
+        stats = client.stats()
+        for k, v in stats.items():
+            print(f"  {k}: {v}")
+
+        # 测试 delete
+        print("\n--- 测试 delete ---")
+        deleted = client.delete(ns, b"https://example.com/1")
+        print(f"删除: {deleted}")  # True
+        print(f"删除后存在: {client.exists(ns, b'https://example.com/1')}")  # False
+
+
+if __name__ == "__main__":
+    main()

dedup_client/dedup_pb2.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: dedup.proto
+# Protobuf Python Version: 7.35.1
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+try:
+    _runtime_version.ValidateProtobufRuntimeVersion(
+        _runtime_version.Domain.PUBLIC,
+        7,
+        35,
+        1,
+        '',
+        'dedup.proto'
+    )
+except Exception:
+    pass
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0b\x64\x65\x64up.proto\x12\x08\x64\x65\x64up.v1\"\\\n\x12\x43heckAndSetRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\x0c\x12\x14\n\x0c\x65xpire_at_ms\x18\x03 \x01(\x04\x12\x10\n\x08metadata\x18\x04 \x01(\x0c\"a\n\x13\x43heckAndSetResponse\x12%\n\x06result\x18\x01 \x01(\x0e\x32\x15.dedup.v1.CheckResult\x12#\n\x05value\x18\x02 \x01(\x0b\x32\x14.dedup.v1.DedupValue\"F\n\x17\x42\x61tchCheckAndSetRequest\x12+\n\x05items\x18\x01 \x03(\x0b\x32\x1c.dedup.v1.CheckAndSetRequest\"J\n\x18\x42\x61tchCheckAndSetResponse\x12.\n\x07results\x18\x01 \x03(\x0b\x32\x1d.dedup.v1.CheckAndSetResponse\"/\n\rExistsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\x0c\" \n\x0e\x45xistsResponse\x12\x0e\n\x06\x65xists\x18\x01 \x01(\x08\"5\n\x12\x42\x61tchExistsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"%\n\x13\x42\x61tchExistsResponse\x12\x0e\n\x06\x65xists\x18\x01 \x03(\x08\",\n\nGetRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\x0c\"A\n\x0bGetResponse\x12\r\n\x05\x66ound\x18\x01 \x01(\x08\x12#\n\x05value\x18\x02 \x01(\x0b\x32\x14.dedup.v1.DedupValue\"/\n\rDeleteRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0b\n\x03key\x18\x02 \x01(\x0c\"!\n\x0e\x44\x65leteResponse\x12\x0f\n\x07\x64\x65leted\x18\x01 \x01(\x08\"5\n\x12\x42\x61tchDeleteRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x0c\n\x04keys\x18\x02 \x03(\x0c\"&\n\x13\x42\x61tchDeleteResponse\x12\x0f\n\x07\x64\x65leted\x18\x01 \x03(\x08\"!\n\x0cStatsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\"\xdb\x01\n\rStatsResponse\x12\x12\n\ntotal_keys\x18\x01 \x01(\x04\x12\x16\n\x0e\x62loom_capacity\x18\x02 \x01(\x04\x12\x18\n\x10\x62loom_insertions\x18\x03 \x01(\x04\x12\x11\n\tbloom_fpp\x18\x04 \x01(\x01\x12\x15\n\rcleaned_total\x18\x05 \x01(\x04\x12\x18\n\x10\x62\x61\x63klog_estimate\x18\x06 \x01(\x04\x12\x15\n\rcurrent_speed\x18\x07 \x01(\x04\x12\x14\n\x0cqps_estimate\x18\x08 \x01(\x04\x12\x13\n\x0b\x65rror_count\x18\t \x01(\x04\"K\n\nDedupValue\x12\x14\n\x0c\x65xpire_at_ms\x18\x01 \x01(\x04\x12\x15\n\rcreated_at_ms\x18\x02 \x01(\x04\x12\x10\n\x08metadata\x18\x03 \x01(\x0c*D\n\x0b\x43heckResult\x12\x0c\n\x08INSERTED\x10\x00\x12\r\n\tDUPLICATE\x10\x01\x12\x18\n\x14\x45XPIRED_AND_REPLACED\x10\x02\x32\xb5\x04\n\x0c\x44\x65\x64upService\x12J\n\x0b\x43heckAndSet\x12\x1c.dedup.v1.CheckAndSetRequest\x1a\x1d.dedup.v1.CheckAndSetResponse\x12Y\n\x10\x42\x61tchCheckAndSet\x12!.dedup.v1.BatchCheckAndSetRequest\x1a\".dedup.v1.BatchCheckAndSetResponse\x12;\n\x06\x45xists\x12\x17.dedup.v1.ExistsRequest\x1a\x18.dedup.v1.ExistsResponse\x12J\n\x0b\x42\x61tchExists\x12\x1c.dedup.v1.BatchExistsRequest\x1a\x1d.dedup.v1.BatchExistsResponse\x12\x32\n\x03Get\x12\x14.dedup.v1.GetRequest\x1a\x15.dedup.v1.GetResponse\x12;\n\x06\x44\x65lete\x12\x17.dedup.v1.DeleteRequest\x1a\x18.dedup.v1.DeleteResponse\x12J\n\x0b\x42\x61tchDelete\x12\x1c.dedup.v1.BatchDeleteRequest\x1a\x1d.dedup.v1.BatchDeleteResponse\x12\x38\n\x05Stats\x12\x16.dedup.v1.StatsRequest\x1a\x17.dedup.v1.StatsResponseb\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'dedup_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  DESCRIPTOR._loaded_options = None
+  _globals['_CHECKRESULT']._serialized_start=1169
+  _globals['_CHECKRESULT']._serialized_end=1237
+  _globals['_CHECKANDSETREQUEST']._serialized_start=25
+  _globals['_CHECKANDSETREQUEST']._serialized_end=117
+  _globals['_CHECKANDSETRESPONSE']._serialized_start=119
+  _globals['_CHECKANDSETRESPONSE']._serialized_end=216
+  _globals['_BATCHCHECKANDSETREQUEST']._serialized_start=218
+  _globals['_BATCHCHECKANDSETREQUEST']._serialized_end=288
+  _globals['_BATCHCHECKANDSETRESPONSE']._serialized_start=290
+  _globals['_BATCHCHECKANDSETRESPONSE']._serialized_end=364
+  _globals['_EXISTSREQUEST']._serialized_start=366
+  _globals['_EXISTSREQUEST']._serialized_end=413
+  _globals['_EXISTSRESPONSE']._serialized_start=415
+  _globals['_EXISTSRESPONSE']._serialized_end=447
+  _globals['_BATCHEXISTSREQUEST']._serialized_start=449
+  _globals['_BATCHEXISTSREQUEST']._serialized_end=502
+  _globals['_BATCHEXISTSRESPONSE']._serialized_start=504
+  _globals['_BATCHEXISTSRESPONSE']._serialized_end=541
+  _globals['_GETREQUEST']._serialized_start=543
+  _globals['_GETREQUEST']._serialized_end=587
+  _globals['_GETRESPONSE']._serialized_start=589
+  _globals['_GETRESPONSE']._serialized_end=654
+  _globals['_DELETEREQUEST']._serialized_start=656
+  _globals['_DELETEREQUEST']._serialized_end=703
+  _globals['_DELETERESPONSE']._serialized_start=705
+  _globals['_DELETERESPONSE']._serialized_end=738
+  _globals['_BATCHDELETEREQUEST']._serialized_start=740
+  _globals['_BATCHDELETEREQUEST']._serialized_end=793
+  _globals['_BATCHDELETERESPONSE']._serialized_start=795
+  _globals['_BATCHDELETERESPONSE']._serialized_end=833
+  _globals['_STATSREQUEST']._serialized_start=835
+  _globals['_STATSREQUEST']._serialized_end=868
+  _globals['_STATSRESPONSE']._serialized_start=871
+  _globals['_STATSRESPONSE']._serialized_end=1090
+  _globals['_DEDUPVALUE']._serialized_start=1092
+  _globals['_DEDUPVALUE']._serialized_end=1167
+  _globals['_DEDUPSERVICE']._serialized_start=1240
+  _globals['_DEDUPSERVICE']._serialized_end=1805
+# @@protoc_insertion_point(module_scope)