web-dc-api 0.0.58 → 0.0.60

package/dist/esm/implements/threaddb/lsstoreds/addr_book.js

@@ -0,0 +1,456 @@
+import { Key } from '../../../node_modules/interface-datastore/dist/src/key.js';
+import * as bytes from 'uint8arrays';
+import { peerIdFromString } from '@libp2p/peer-id';
+import { multiaddr } from '@multiformats/multiaddr';
+import { AddrBookRecord } from '../pb/lstore.js';
+import { uniqueLogIds, uniqueThreadIds, dsThreadKey, DefaultOpts } from './global.js';
+import { LRUCache } from 'lru-cache';
+import { ThreadID } from '@textile/threads-id';
+
+// TTL写入模式  
+var TTLWriteMode;
+(function (TTLWriteMode) {
+  TTLWriteMode[TTLWriteMode["Override"] = 0] = "Override";
+  TTLWriteMode[TTLWriteMode["Extend"] = 1] = "Extend";
+})(TTLWriteMode || (TTLWriteMode = {}));
+// NewAddrBook initializes a new datastore-backed address book. It serves as a drop-in replacement for pstoremem
+// (memory-backed peerstore), and works with any datastore implementing the ds.Batching interface.
+//
+// Threads and logs addresses are serialized into protobuf, storing one datastore entry per (thread, log), along with metadata
+// to control address expiration. To alleviate disk access and serde overhead, we internally use a read/write-through
+// ARC cache, the size of which is adjustable via Options.CacheSize.
+async function newAddrBook(ds, opts) {
+  if (!opts) {
+    opts = DefaultOpts();
+  }
+  const ab = new DsAddrBook(ds, opts);
+  if (opts.CacheSize > 0) {
+    ab.cache = new LRUCache({
+      max: opts.CacheSize
+    });
+  } else {
+    ab.cache = new LRUCache({
+      max: 0
+    });
+  }
+  return ab;
+}
+class DsAddrBook {
+  constructor(ds, opts) {
+    Object.defineProperty(this, "opts", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    Object.defineProperty(this, "ds", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    Object.defineProperty(this, "cache", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    this.opts = opts;
+    this.ds = ds;
+    this.cache = new LRUCache({
+      max: opts.CacheSize || 0
+    });
+  }
+  async addAddr(t, p, addrs, ttl) {
+    if (ttl <= 0) return;
+    const cleaned = this.cleanAddrs([addrs]);
+    await this.setAddrs(t, p, cleaned, ttl);
+  }
+  async addAddrs(t, p, addrs, ttl) {
+    if (ttl <= 0) return;
+    const cleaned = this.cleanAddrs(addrs);
+    await this.setAddrs(t, p, cleaned, ttl);
+  }
+  cleanAddrs(addrs) {
+    return addrs.reduce((acc, item) => {
+      try {
+        if (!item) return acc;
+        const addr = typeof item === 'string' ? multiaddr(item) : item instanceof multiaddr ? item : null;
+        return addr ? [...acc, addr] : acc;
+      } catch {
+        return acc;
+      }
+    }, []);
+  }
+  async setAddr(t, p, addr, ttl) {
+    if (ttl <= 0) return;
+    const cleaned = this.cleanAddrs([addr]);
+    await this.setAddrs(t, p, cleaned, ttl);
+  }
+  async setAddrs(t, logId, addrs, ttl) {
+    let release = () => {};
+    try {
+      const record = await this.loadRecord(t, logId.toString(), true, false);
+      release = await record.lock.acquire();
+      const newExp = Date.now() + ttl;
+      const existed = new Array(addrs.length).fill(false);
+      // 更新现有地址  
+      outer: for (const [i, addr] of addrs.entries()) {
+        for (const entry of record.data.addrs) {
+          if (addr.bytes == entry.addr) {
+            existed[i] = true;
+            entry.ttl = ttl;
+            entry.expiry = newExp;
+            continue outer;
+          }
+        }
+      }
+      // 添加新地址  
+      const added = addrs.filter((_, i) => !existed[i]).map(addr => new AddrBookRecord.AddrEntry({
+        addr: addr.bytes,
+        expiry: newExp,
+        ttl
+      }));
+      record.data.addrs.push(...added);
+      record.dirty = true;
+      await this.cleanRecord(record);
+      await this.flushRecord(record);
+    } finally {
+      release();
+    }
+  }
+  async updateAddrs(t, logId, oldTTL, newTTL) {
+    let release = () => {};
+    try {
+      const record = await this.loadRecord(t, logId.toString(), true, false);
+      release = await record.lock.acquire();
+      const newExp = Date.now() + newTTL;
+      for (const entry of record.data.addrs) {
+        if (entry.ttl === oldTTL) {
+          entry.ttl = newTTL;
+          entry.expiry = newExp;
+          record.dirty = true;
+        }
+      }
+      await this.cleanRecord(record);
+      await this.flushRecord(record);
+    } finally {
+      release();
+    }
+  }
+  async addrs(t, logId) {
+    let release = () => {};
+    try {
+      const record = await this.loadRecord(t, logId.toString(), true, true);
+      release = await record.lock.acquire();
+      const addrs = [];
+      for (const entry of record.data.addrs) {
+        addrs.push(multiaddr(entry.addr));
+      }
+      return addrs;
+    } finally {
+      release();
+    }
+  }
+  async clearAddrs(t, logId) {
+    this.cache.delete({
+      threadID: t,
+      peerID: logId.toString()
+    });
+    const key = this.genDSKey(t, logId.toString());
+    try {
+      await this.ds.delete(key);
+      await this.invalidateEdge(t);
+    } catch (err) {
+      throw new Error(`Failed to clear addresses for log ${logId.toString()}: ${err.message}`);
+    }
+  }
+  async logsWithAddrs(t) {
+    try {
+      const ids = await uniqueLogIds(this.ds, DsAddrBook.logBookBase.child(new Key(t.toString())), result => {
+        return result.name();
+      });
+      return ids;
+    } catch (err) {
+      throw new Error(`Error while retrieving logs with addresses: ${err.message}`);
+    }
+  }
+  async threadsFromAddrs() {
+    try {
+      const ids = await uniqueThreadIds(this.ds, DsAddrBook.logBookBase, result => {
+        return result.parent().name();
+      });
+      return ids;
+    } catch (err) {
+      throw new Error(`Error while retrieving thread from addresses: ${err.message}`);
+    }
+  }
+  async addrsEdge(t) {
+    const key = this.genDSKey(t, DsAddrBook.logBookEdge.toString());
+    try {
+      const value = await this.ds.get(key);
+      return Number(new DataView(value.buffer).getBigUint64(0));
+    } catch (err) {
+      if (err.code !== 'ERR_NOT_FOUND') {
+        throw err;
+      }
+    }
+    const result = this.ds.query({
+      prefix: this.genDSKey(t, DsAddrBook.logBookBase.toString()).toString()
+    });
+    const now = Date.now();
+    const as = [];
+    for await (const entry of result) {
+      const {
+        tid,
+        pid,
+        record
+      } = this.decodeAddrEntry(entry, true);
+      if (!record) {
+        continue;
+      }
+      for (const addr of record.addrs) {
+        if (addr.expiry > now) {
+          as.push({
+            logId: pid,
+            addr: addr.addr
+          });
+        }
+      }
+    }
+    if (as.length === 0) {
+      throw new Error('Thread not found');
+    }
+    const edge = this.computeAddrsEdge(as);
+    const buff = new ArrayBuffer(8);
+    new BigUint64Array(buff)[0] = BigInt(edge);
+    await this.ds.put(key, new Uint8Array(buff));
+    return edge;
+  }
+  async loadRecord(t, p, cache, update) {
+    const cacheKey = {
+      threadID: t,
+      peerID: p
+    };
+    if (this.cache.has(cacheKey)) {
+      const addrsRecord = this.cache.get(cacheKey);
+      await this.cleanRecord(addrsRecord);
+      if (update) {
+        await this.flushRecord(addrsRecord);
+      }
+      return addrsRecord;
+    }
+    const key = this.genDSKey(t, p);
+    try {
+      const data = await this.ds.get(key);
+      const record = AddrBookRecord.deserialize(data);
+      const addrsRecord = {
+        lock: new AsyncMutex(),
+        data: record,
+        dirty: false
+      };
+      await this.cleanRecord(addrsRecord);
+      if (update) {
+        await this.flushRecord(addrsRecord);
+      }
+      if (cache) {
+        this.cache.set(cacheKey, addrsRecord);
+        return addrsRecord;
+      }
+      return addrsRecord;
+    } catch (err) {
+      if (err.code === 'ERR_NOT_FOUND') {
+        const peerIdBytes = new TextEncoder().encode(p);
+        return {
+          lock: new AsyncMutex(),
+          data: AddrBookRecord.fromObject({
+            threadID: t.toBytes(),
+            peerID: peerIdBytes,
+            addrs: []
+          }),
+          dirty: false
+        };
+      }
+      throw err;
+    }
+  }
+  async cleanRecord(record) {
+    const now = Date.now();
+    const valid = record.data.addrs.filter(a => a.expiry > now);
+    const changed = valid.length !== record.data.addrs.length;
+    record.data.addrs = valid.sort((a, b) => a.expiry - b.expiry);
+    return changed;
+  }
+  async flushRecord(record) {
+    const threadId = ThreadID.fromBytes(record.data.threadID);
+    const logId = new TextDecoder().decode(record.data.peerID);
+    if (record.data.addrs.length === 0) {
+      await this.ds.delete(this.genDSKey(threadId, logId));
+    } else {
+      const data = record.data.serialize();
+      await this.ds.put(this.genDSKey(threadId, logId), data);
+    }
+    record.dirty = false;
+  }
+  genDSKey(t, p) {
+    const tKey = new Key(t.toString());
+    const pKey = new Key(p);
+    return DsAddrBook.logBookBase.child(tKey).child(pKey);
+  }
+  async invalidateEdge(tid) {
+    var key = dsThreadKey(tid, DsAddrBook.logBookEdge);
+    return this.ds.delete(key);
+  }
+  async traverse(withAddrs) {
+    const data = {};
+    let result = this.ds.query({
+      prefix: DsAddrBook.logBookBase.toString()
+    });
+    for await (const entry of result) {
+      const {
+        tid,
+        pid,
+        record
+      } = this.decodeAddrEntry(entry, withAddrs);
+      const tidStr = tid.toString();
+      if (!data[tidStr]) {
+        data[tidStr] = {};
+      }
+      data[tidStr][pid.toString()] = record;
+    }
+    return data;
+  }
+  decodeAddrEntry(entry, withAddrs) {
+    const kns = entry.key.toString().split('/');
+    if (kns.length < 3) {
+      throw new Error(`bad addressbook key detected: ${entry.key}`);
+    }
+    // get thread and log IDs from the key components
+    const ts = kns[kns.length - 2];
+    const ls = kns[kns.length - 1];
+    const tid = ThreadID.fromString(ts);
+    const pid = peerIdFromString(ls);
+    if (withAddrs) {
+      const record = AddrBookRecord.deserialize(entry.value);
+      return {
+        tid,
+        pid,
+        record
+      };
+    }
+    return {
+      tid,
+      pid
+    };
+  }
+  computeAddrsEdge(as) {
+    // sort peer addresses for deterministic edge computation
+    as.sort((a, b) => {
+      if (a.logId.toString() === b.logId.toString()) {
+        return bytes.compare(a.addr, b.addr);
+      }
+      return a.logId.toString() < b.logId.toString() ? -1 : 1;
+    });
+    const hasher = new TextEncoder();
+    let hash = 0n;
+    for (const item of as) {
+      const logIdBytes = hasher.encode(item.logId.toString());
+      const addrBytes = item.addr;
+      for (const byte of logIdBytes) {
+        hash = (hash * 31n + BigInt(byte)) % 2n ** 64n;
+      }
+      for (const byte of addrBytes) {
+        hash = (hash * 31n + BigInt(byte)) % 2n ** 64n;
+      }
+    }
+    return Number(hash);
+  }
+  // dumpAddrs(): Promise<DumpAddrBook>;  
+  async dumpAddrs() {
+    const dump = {};
+    const data = await this.traverse(true);
+    for (const [tid, logs] of Object.entries(data)) {
+      const lm = {};
+      for (const [lid, rec] of Object.entries(logs)) {
+        if (rec.addrs.length > 0) {
+          const addrs = rec.addrs.map(r => ({
+            addr: multiaddr(r.addr),
+            expires: new Date(r.expiry)
+          }));
+          lm[lid] = addrs.map(a => ({
+            addr: a.addr,
+            expires: a.expires
+          }));
+        }
+      }
+      dump[tid] = lm;
+    }
+    return {
+      data: dump
+    };
+  }
+  async restoreAddrs(dump) {
+    if (Object.keys(dump.data).length === 0) {
+      throw new Error('Empty dump');
+    }
+    const stored = await this.traverse(false);
+    for (const [tid, logs] of Object.entries(stored)) {
+      for (const lid of Object.keys(logs)) {
+        await this.clearAddrs(ThreadID.fromString(tid), peerIdFromString(lid));
+      }
+    }
+    const current = new Date();
+    for (const [tid, logs] of Object.entries(dump.data)) {
+      for (const [lid, addrs] of Object.entries(logs)) {
+        for (const addr of addrs) {
+          const ttl = addr.expires.getTime() - current.getTime();
+          if (ttl > 0) {
+            await this.setAddrs(ThreadID.fromString(tid), peerIdFromString(lid), [multiaddr(addr.addr)], ttl);
+          }
+        }
+      }
+    }
+  }
+}
+Object.defineProperty(DsAddrBook, "logBookBase", {
+  enumerable: true,
+  configurable: true,
+  writable: true,
+  value: new Key('/thread/addrs')
+});
+Object.defineProperty(DsAddrBook, "logBookEdge", {
+  enumerable: true,
+  configurable: true,
+  writable: true,
+  value: new Key('/thread/addrs:edge')
+});
+// 辅助类实现  
+class AsyncMutex {
+  constructor() {
+    Object.defineProperty(this, "queue", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: []
+    });
+    Object.defineProperty(this, "locked", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: false
+    });
+  }
+  async acquire() {
+    while (this.locked) {
+      await new Promise(resolve => this.queue.push(resolve));
+    }
+    this.locked = true;
+    return () => {
+      this.locked = false;
+      const next = this.queue.shift();
+      next?.();
+    };
+  }
+}
+
+export { DsAddrBook, newAddrBook };

package/dist/esm/implements/threaddb/lsstoreds/global.js

@@ -0,0 +1,112 @@
+import { ThreadID } from '@textile/threads-id';
+import { peerIdFromString } from '@libp2p/peer-id';
+import { Key } from '../../../node_modules/interface-datastore/dist/src/key.js';
+
+// 默认配置生成器  
+function DefaultOpts() {
+  return {
+    CacheSize: 1024,
+    GCPurgeInterval: 2 * 60 * 60 * 1000,
+    // 2小时  
+    GCInitialDelay: 60 * 1000 // 60秒  
+  };
+}
+// 唯一线程ID提取  
+async function uniqueThreadIds(ds, prefix, extractor) {
+  const q = {
+    prefix: prefix.toString()
+  };
+  const results = await ds.queryKeys(q);
+  const idSet = new Set();
+  for await (const result of results) {
+    idSet.add(extractor(result));
+  }
+  return Array.from(idSet).map(id => parseThreadID(id)).filter(Boolean);
+}
+// 唯一日志ID提取  
+async function uniqueLogIds(ds, prefix, extractor) {
+  const q = {
+    prefix: prefix.toString()
+  };
+  const results = ds.queryKeys(q);
+  const idSet = new Set();
+  for await (const result of results) {
+    idSet.add(extractor(result));
+  }
+  const parsedIds = await Promise.all(Array.from(idSet).map(async id => {
+    try {
+      return await parseLogID(id);
+    } catch {
+      return null;
+    }
+  }));
+  return parsedIds.filter(Boolean);
+}
+function parseThreadID(id) {
+  try {
+    return ThreadID.fromString(id);
+  } catch {
+    return undefined;
+  }
+}
+async function parseLogID(id) {
+  try {
+    return peerIdFromString(id);
+  } catch {
+    throw new Error('Invalid log ID format');
+  }
+}
+function dsThreadKey(t, baseKey) {
+  return baseKey.child(new Key(t.toString()));
+}
+function dsLogKey(t, p, baseKey) {
+  return baseKey.child(new Key(t.toString())).child(new Key(p.toString()));
+}
+function uvError(n) {
+  return n <= 0 ? new Error('Invalid varint') : null;
+}
+function getVersion(data) {
+  const [vers, n] = binaryUvarint(data);
+  const err = uvError(n);
+  return [vers, n, err];
+}
+function binaryUvarint(data) {
+  let x = 0;
+  let s = 0;
+  for (let i = 0; i < data.length; i++) {
+    const b = data[i];
+    if (b < 0x80) {
+      if (i > 9 || i === 9 && b > 1) {
+        return [0, -(i + 1)]; // overflow
+      }
+      return [x | b << s, i + 1];
+    }
+    x |= (b & 0x7f) << s;
+    s += 7;
+  }
+  return [0, 0];
+}
+function validateIDData(data) {
+  const [vers, n, err] = getVersion(data);
+  if (err) {
+    return err;
+  }
+  if (vers !== 1) {
+    return new Error(`expected 1 as the id version number, got: ${vers}`);
+  }
+  const [variant, cn] = binaryUvarint(data.subarray(n));
+  const variantErr = uvError(cn);
+  if (variantErr) {
+    return variantErr;
+  }
+  if (variant !== 0 && variant !== 1) {
+    return new Error(`expected Raw or AccessControlled as the id variant, got: ${variant}`);
+  }
+  const id = data.subarray(n + cn);
+  if (id.length === 0) {
+    return new Error('expected random id bytes but there are none');
+  }
+  return null;
+}
+
+export { DefaultOpts, dsLogKey, dsThreadKey, parseLogID, parseThreadID, uniqueLogIds, uniqueThreadIds, validateIDData };