web-dc-api 0.1.5 → 0.1.7

package/lib/implements/threaddb/lsstoreds/addr_book.ts

@@ -1,549 +0,0 @@
-import { Datastore, Key, Pair } from 'interface-datastore'
-import * as bytes from 'uint8arrays'
-import {  peerIdFromString } from '@libp2p/peer-id'  
-import type { PeerId,PrivateKey,PublicKey } from "@libp2p/interface";
-import { Multiaddr, multiaddr, isMultiaddr } from '@multiformats/multiaddr'  
-import {  AddrBook,DumpAddrBook} from '../core/logstore' // 核心接口定义  
-import  * as pb from '../pb/lstore'
-import {uniqueLogIds,uniqueThreadIds,DSOptions,AllowEmptyRestore,dsThreadKey} from './global'
-import {LRUCache} from 'lru-cache'  
-import { ThreadID } from '@textile/threads-id';
-import {DefaultOpts} from './global';
-import { TxnDatastoreExtended, Transaction } from '../core/db';
-import * as buffer from "buffer/";
-import { bases } from 'multiformats/basics';
-const { Buffer } = buffer;
-
-const ADDR_FNV_OFFSET_BASIS = 0xcbf29ce484222325n;
-const ADDR_FNV_PRIME = 0x100000001b3n;
-const ADDR_MOD64 = 2n ** 64n;
-
-
-// 缓存键类型  
-type CacheKey = {  
-  threadID: ThreadID  
-  peerID: string  
-}  
-
-// TTL写入模式  
-enum TTLWriteMode {  
-  Override,  
-  Extend  
-}  
-
-// 地址记录增强类型  
-interface  AddrsRecord {  
-  lock: AsyncMutex  
-  data: pb.AddrBookRecord   
-  dirty: boolean  
-}  
-
-
-// NewAddrBook initializes a new datastore-backed address book. It serves as a drop-in replacement for pstoremem
-// (memory-backed peerstore), and works with any datastore implementing the ds.Batching interface.
-//
-// Threads and logs addresses are serialized into protobuf, storing one datastore entry per (thread, log), along with metadata
-// to control address expiration. To alleviate disk access and serde overhead, we internally use a read/write-through
-// ARC cache, the size of which is adjustable via Options.CacheSize.
-export async function newAddrBook(ds: Datastore, opts?: DSOptions): Promise<DsAddrBook> {
-    if (!opts) {
-        opts = DefaultOpts();
-    }
-    const ab = new DsAddrBook(ds, opts);
-    if (opts.CacheSize > 0) {
-        ab.cache = new LRUCache({ max: opts.CacheSize });
-    } else {
-        ab.cache = new LRUCache({ max: 0 });
-    }
-    return ab;
-}
-
-
-export class DsAddrBook implements AddrBook {  
-  private static readonly logBookBase = new Key('/thread/addrs')  
-  private static readonly logBookEdge = new Key('/thread/addrs:edge')  
- 
- 
-  private opts: DSOptions  
-  private ds: TxnDatastoreExtended  
-   cache: LRUCache<CacheKey, AddrsRecord>  
-
-  constructor( ds: Datastore, opts: DSOptions) {  
-    this.opts = opts  
-    this.ds = ds as TxnDatastoreExtended  
-    this.cache = new LRUCache({ max: opts.CacheSize || 0 })   
-  }  
-
-
-    async addAddr(t: ThreadID, p: PeerId, addrs: Multiaddr, ttl: number): Promise<void> {  
-
-        if (ttl <= 0) return  
-        const cleaned = this.cleanAddrs([addrs])
-        await this.setAddrs(t, p, cleaned, ttl)
-    }
-
-  async addAddrs(t: ThreadID, p: PeerId, addrs: Multiaddr[], ttl: number): Promise<void> {  
-    if (ttl <= 0) return  
-    const cleaned = this.cleanAddrs(addrs)  
-    await this.setAddrs(t, p, cleaned, ttl)  
-  }   
-
-   cleanAddrs(addrs: (string | Multiaddr | null | undefined)[]): Multiaddr[] {  
-    return addrs.reduce<Multiaddr[]>((acc, item) => {  
-      try {  
-        if (!item) return acc 
-        const addr = typeof item === 'string'   
-          ?  multiaddr(item)   
-          : (isMultiaddr(item) ? item : null)  
-        return addr ? [...acc, addr] : acc  
-      } catch {  
-        return acc 
-      }  
-    }, [])  
-  }
-
-
-    async setAddr(t: ThreadID, p: PeerId, addr: Multiaddr, ttl: number): Promise<void> {  
-        if (ttl <= 0) return  
-        const cleaned = this.cleanAddrs([addr])  
-        await this.setAddrs(t, p, cleaned, ttl)  
-    }
-
-   async setAddrs(t: ThreadID, logId: PeerId, addrs: Multiaddr[], ttl: number): Promise<void> {  
-    let release = () => {}
-    try {  
-      const record = await this.loadRecord(t, logId.toString(), true,false)  
-      release = await record.lock.acquire()  
-      const newExp = Date.now() + ttl  
-      const existed = new Array(addrs.length).fill(false)  
-
-      // 更新现有地址  
-      outer: for (const [i, addr] of addrs.entries()) {  
-        for (const entry of record.data.addrs) {  
-          if (bytes.compare(addr.bytes, entry.addr) === 0) {
-            existed[i] = true  
-            entry.ttl = ttl  
-            entry.expiry = newExp  
-            continue outer  
-          }  
-        }  
-      }  
-
-      // 添加新地址  
-      const added = addrs  
-        .filter((_, i) => !existed[i])  
-        .map(addr => new  pb.AddrBookRecord.AddrEntry({
-            addr: addr.bytes,  
-            expiry: newExp,  
-            ttl  
-        }))
-
-      record.data.addrs.push(...added)  
-      record.dirty = true  
-      await this.cleanRecord(record)  
-      await this.flushRecord(record)  
-      await this.invalidateEdge(t)
-    } finally {  
-      release()  
-    }  
-  }  
-
-  async updateAddrs(t: ThreadID, logId: PeerId, oldTTL: number, newTTL: number): Promise<void> {
-    let release = () => {}
-    try {  
-        const record = await this.loadRecord(t, logId.toString(), true,false)  
-        release = await record.lock.acquire()  
-        const newExp = Date.now() + newTTL
-        for (const entry of record.data.addrs) {  
-            if (entry.ttl === oldTTL) {  
-            entry.ttl = newTTL  
-            entry.expiry = newExp  
-            record.dirty = true  
-            }  
-        }
-        await this.cleanRecord(record)
-        await this.flushRecord(record)
-        await this.invalidateEdge(t)
-    } finally {
-        release()
-    }
-	
-}
-
-
-async addrs(t: ThreadID, logId: PeerId): Promise<Multiaddr[]> {
-    let release = () => {}
-    try {  
-        const record = await this.loadRecord(t, logId.toString(), true,true)  
-        release = await record.lock.acquire()  
-        const addrs: Multiaddr[] = []
-        for (const entry of record.data.addrs) {  
-            addrs.push(multiaddr(entry.addr))
-        }
-        return addrs
-    } finally {
-        release()
-    }
-}
-
-
-async clearAddrs(t: ThreadID, logId: PeerId): Promise<void> {
-    this.cache.delete({ threadID: t, peerID: logId.toString() });
-
-    const key = this.genDSKey(t, logId.toString());
-    try {
-        await this.ds.delete(key);
-        await this.invalidateEdge(t);
-    } catch (err:any) {
-        throw new Error(`Failed to clear addresses for log ${logId.toString()}: ${err.message}`);
-    }
-}
-
-
-
-async logsWithAddrs(t: ThreadID): Promise<PeerId[]> {
-    try {
-        const ids = await uniqueLogIds(this.ds, DsAddrBook.logBookBase.child(new Key(t.toString())), (result) => {
-            return result.name();
-        });
-        return ids;
-    } catch (err : any) {
-        throw new Error(`Error while retrieving logs with addresses: ${err.message}`);
-    }
-}
-
-async threadsFromAddrs(): Promise<ThreadID[]> {
-    try {
-        const ids = await uniqueThreadIds(this.ds, DsAddrBook.logBookBase, (result) => {
-            return result.parent().name();
-        });
-        return ids;
-    } catch (err:any) {
-        throw new Error(`Error while retrieving thread from addresses: ${err.message}`);
-    }
-}
-
-
-
-
-    async addrsEdge(t: ThreadID, exceptPeerId?: string): Promise<bigint> {
-        const key = dsThreadKey(t, DsAddrBook.logBookEdge);
-        const retries = 3
-        for (let attempt = 1; attempt <= retries; attempt++) {
-            try {
-                return await this.calculateEdge(t, key, exceptPeerId);
-            } catch (err: any)  {
-                if (err.code !== 'TX_CONFLICT') throw err;
-                await this.randomDelay(attempt);
-            }
-        }
-        throw new Error('Edge computation failed');
-    }
-
-    private async calculateEdge(t: ThreadID, key: Key, exceptPeerId?: string): Promise<bigint> {
-        return this.withTransaction(async txn => {
-            try {
-                const data = await txn.get(key);
-                if (data) {
-                    return this.decodeStoredEdge(data);
-                }
-            } catch (err: any)  {
-                if (err.code !== 'ERR_NOT_FOUND') throw err;
-            }
-
-            const addrs = await this.collectAddrs(txn, t, exceptPeerId);
-            if (addrs.length === 0) throw new Error('Thread not found');
-
-            const edge = this.computeAddrsEdge(addrs);
-            const buffer = Buffer.alloc(8);
-            this.writeEdgeValue(buffer, edge);
-
-            await txn.put(key, buffer);
-            return edge;
-        });
-    }
-
-    private async collectAddrs(txn: Transaction, t: ThreadID, exceptPeerId?: string): Promise<{ peerID: PeerId, addr: Uint8Array }[]> {
-        const query = { prefix: dsThreadKey(t, DsAddrBook.logBookBase).toString() };
-        const result = txn.query(query);
-        const now = Date.now();
-        const as: { peerID: PeerId, addr: Uint8Array }[] = [];
-
-        for await (const entry of result) {
-            const rawKey = entry.key as any;
-            const key = rawKey instanceof Key ? rawKey : new Key(rawKey.toString());
-            const pair: Pair = { key, value: entry.value };
-            const { tid,pid, record } = this.decodeAddrEntry(pair, true);
-            if (!record || (exceptPeerId && pid.toString() == exceptPeerId)) {
-                continue;
-            }
-            for (const addr of record.addrs) {
-                if (addr.expiry > now) {
-                    as.push({ peerID:pid, addr: addr.addr });
-                }
-            }
-        }
-        return as;
-    }
-
-    private async withTransaction<T>(fn: (txn: Transaction) => Promise<T>): Promise<T> {
-        const txn = await this.ds.newTransactionExtended(false);
-        try {
-            const result = await fn(txn);
-            await txn.commit();
-            return result;
-        } catch (err: any) {
-            txn.discard();
-            throw err;
-        }
-    }
-
-    private decodeStoredEdge(data: Uint8Array): bigint {
-        const buf: any = Buffer.from(data);
-        const reader = buf as unknown as { readBigUInt64BE?: (offset?: number) => bigint };
-        if (buf.length >= 8 && typeof reader.readBigUInt64BE === 'function') {
-            return reader.readBigUInt64BE(0);
-        }
-        if (buf.length >= 4) {
-            return BigInt(buf.readUInt32BE(0));
-        }
-        throw new Error('Corrupted head edge value');
-    }
-
-    private writeEdgeValue(buf: any, value: bigint): void {
-        const writer = buf as unknown as { writeBigUInt64BE?: (val: bigint, offset?: number) => number };
-        if (typeof writer.writeBigUInt64BE === 'function') {
-            writer.writeBigUInt64BE(value, 0);
-            return;
-        }
-        buf.writeUInt32BE(Number(value & 0xffffffffn), 0);
-    }
-
-    private randomDelay(attempt: number): Promise<void> {
-        const delay = 50 * attempt + Math.random() * 30;
-        return new Promise(resolve =>
-            setTimeout(resolve, delay)
-        );
-    }
-
-
-
-  private async loadRecord(t: ThreadID, p: string, cache: boolean,update:boolean): Promise<AddrsRecord> {  
-    const cacheKey = { threadID: t, peerID: p }  
-    if (this.cache.has(cacheKey)) {  
-        const addrsRecord = this.cache.get(cacheKey)!
-        await this.cleanRecord(addrsRecord)
-        if (update) {
-            await this.flushRecord(addrsRecord)
-        }
-        return addrsRecord
-    }
-    const key = this.genDSKey(t, p)  
-    try {  
-        const data = await this.ds.get(key)  
-        const record =  pb.AddrBookRecord.deserialize(data)
-        const addrsRecord = { lock: new AsyncMutex(), data: record, dirty: false }
-        await this.cleanRecord(addrsRecord)
-        if (update) {
-            await this.flushRecord(addrsRecord)
-        }
-        if (cache) {
-            this.cache.set(cacheKey, addrsRecord)
-            return addrsRecord
-        }   
-        return addrsRecord 
-    } catch (err:any) {  
-      if (err.code === 'ERR_NOT_FOUND') {  
-        const peerIdBytes = new TextEncoder().encode(p)
-        return {  
-          lock: new AsyncMutex(),  
-          data: pb.AddrBookRecord.fromObject({threadID:t.toBytes(),  peerID:peerIdBytes,addrs:[] } ),  
-          dirty: false  
-        }  
-      }  
-      throw err  
-    }  
-  }  
-
-  private async cleanRecord(record: AddrsRecord): Promise<boolean> {  
-    const now = Date.now()  
-    const valid = record.data.addrs.filter(a => a.expiry > now)  
-    const changed = valid.length !== record.data.addrs.length  
-    record.data.addrs = valid.sort((a, b) => a.expiry - b.expiry)  
-    return changed  
-  }  
-
-  private async flushRecord(record: AddrsRecord): Promise<void> {  
-    const threadId = ThreadID.fromBytes(record.data.threadID)
-    const logId = new TextDecoder().decode(record.data.peerID)
-    if (record.data.addrs.length === 0) {  
-        await this.ds.delete(this.genDSKey(threadId, logId))
-    } else {  
-      const data = record.data.serialize()
-      await this.ds.put(this.genDSKey(threadId, logId), data)  
-    }  
-    record.dirty = false  
-  }  
-
-  private genDSKey(t: ThreadID, p: string): Key {  
-    const tKey = new Key(t.toString()) 
-    const pKey = new Key(p) 
-    return DsAddrBook.logBookBase.child(tKey).child(pKey)  
-  }  
-
-
-  private async invalidateEdge(tid: ThreadID): Promise<void> {
-	var key = dsThreadKey(tid, DsAddrBook.logBookEdge)
-	return this.ds.delete(key)
-  }
-
-
-
-private async traverse(withAddrs: boolean): Promise<{ [key: string]: { [key: string]: pb.AddrBookRecord } }> {
-    const data: { [key: string]: { [key: string]: pb.AddrBookRecord } } = {};
-    let result = this.ds.query({ prefix: DsAddrBook.logBookBase.toString()});
-    for await (const entry of result) {
-        const { tid, pid, record } = this.decodeAddrEntry(entry, withAddrs);
-        const tidStr = tid.toString();
-        if (!data[tidStr]) {
-            data[tidStr] = {};
-        }
-        data[tidStr][pid.toString()] = record!;
-    }
-    return data;
-}
-
-
-    private decodeAddrEntry(entry: any, withAddrs: boolean): { tid: ThreadID, pid: PeerId, record?: pb.AddrBookRecord } {
-        const keyStr = typeof entry.key === 'string' ? entry.key : entry.key.toString();
-        const kns = keyStr.split('/');
-        if (kns.length < 3) {
-            throw new Error(`bad addressbook key detected: ${keyStr}`);
-        }
-
-        // get thread and log IDs from the key components
-        const ts = kns[kns.length - 2];
-        const ls = kns[kns.length - 1];
-
-        const tid = ThreadID.fromString(ts!);
-        const pid = peerIdFromString(ls!);
-
-        if (withAddrs) {
-            const record = pb.AddrBookRecord.deserialize(entry.value);
-            return { tid, pid, record };
-        }
-
-        return { tid, pid };
-    }
-
-      
-
-    private computeAddrsEdge(as: { peerID: PeerId, addr: Uint8Array }[]): bigint {
-        const entries = as.map(item => ({
-            peerIDBytes: this.decodePeerId(item.peerID.toString(), new TextEncoder()),
-            addr: item.addr,
-        }));
-
-        entries.sort((a, b) => {
-            const peerIDCompare = bytes.compare(a.peerIDBytes, b.peerIDBytes);
-            if (peerIDCompare !== 0) {
-                return peerIDCompare;
-            }
-            return bytes.compare(a.addr, b.addr);
-        });
-
-        let hash = ADDR_FNV_OFFSET_BASIS;
-
-        for (const entry of entries) {
-            hash = this.fnv1a64(entry.peerIDBytes, hash);
-            hash = this.fnv1a64(entry.addr, hash);
-        }
-
-        return hash;
-    }
-
- private decodePeerId(peerID: string, encoder: TextEncoder): Uint8Array {
-        if (!peerID) {
-            return new Uint8Array();
-        }
-        try {
-            return bases.base58btc.baseDecode(peerID);
-        } catch (err: any) {
-            return encoder.encode(peerID);
-        }
-    }
-
-    private fnv1a64(data: Uint8Array, initial: bigint): bigint {
-        let hash = initial;
-        for (const byte of data) {
-            hash ^= BigInt(byte);
-            hash = (hash * ADDR_FNV_PRIME) % ADDR_MOD64;
-        }
-        return hash;
-    }
-
-
-    // dumpAddrs(): Promise<DumpAddrBook>;  
-    async dumpAddrs(): Promise<DumpAddrBook> {
-        const dump: { [key: string]: { [key: string]: { addr: Multiaddr, expires: Date }[] } } = {};
-        const data = await this.traverse(true);
-
-        for (const [tid, logs] of Object.entries(data)) {
-            const lm: { [key: string]: { addr: Multiaddr, expires: Date }[] } = {};
-            for (const [lid, rec] of Object.entries(logs)) {
-                if (rec.addrs.length > 0) {
-                    const addrs = rec.addrs.map(r => ({
-                        addr: multiaddr(r.addr),
-                        expires: new Date(r.expiry)
-                    }));
-                    lm[lid] = addrs.map(a => ({ addr: a.addr, expires: a.expires }));
-                }
-            }
-            dump[tid] = lm;
-        }
-        return { data: dump };
-    }
-
-    async restoreAddrs(dump:DumpAddrBook): Promise<void> {
-        if (!AllowEmptyRestore && Object.keys(dump.data).length === 0) {
-            throw new Error('Empty dump');
-        }
-
-        const stored = await this.traverse(false);
-
-        for (const [tid, logs] of Object.entries(stored)) {
-            for (const lid of Object.keys(logs)) {
-                await this.clearAddrs(ThreadID.fromString(tid), peerIdFromString(lid));
-            }
-        }
-        const current = new Date();
-        for (const [tid, logs] of Object.entries(dump.data)) {
-            for (const [lid, addrs] of Object.entries(logs)) {
-                for (const addr of addrs) {
-                    const ttl = addr.expires.getTime() - current.getTime();
-                    if (ttl > 0) {
-                        await this.setAddrs(ThreadID.fromString(tid), peerIdFromString(lid), [multiaddr(addr.addr)], ttl);
-                    }
-                }
-            }
-        }
-    }
-}
-
-// 辅助类实现  
-class AsyncMutex {  
-  private queue: (() => void)[] = []  
-  private locked = false  
-
-  async acquire(): Promise<() => void> {  
-    while (this.locked) {  
-      await new Promise<void>(resolve => this.queue.push(resolve))  
-    }  
-    this.locked = true  
-    return () => {  
-      this.locked = false  
-      const next = this.queue.shift()  
-      next?.()  
-    }  
-  }  
-}

package/lib/implements/threaddb/lsstoreds/cache.ts

@@ -1,36 +0,0 @@
-// 修改后的缓存接口声明（保证方法签名严格匹配）  
-interface CacheStore<K, V> {  
-    get(key: K): V | undefined;  
-    add(key: K, value: V): void;  // 明确声明两个参数  
-    remove(key: K): void;  
-    contains(key: K): boolean;  
-    peek(key: K): V | undefined;  
-    keys(): K[];  
-  }  
-  
-  // 修正实现类的参数匹配  
-  class NoopCache<K, V> implements CacheStore<K, V> {  
-    get(_key: K): V | undefined {  
-      return undefined;  
-    }  
-  
-    add(_key: K, _value: V): void {  
-      // 空操作，保留两个参数  
-    }  
-  
-    remove(_key: K): void {  
-      // 空操作，保留单个参数  
-    }  
-  
-    contains(_key: K): boolean {  
-      return false;  
-    }  
-  
-    peek(_key: K): V | undefined {  
-      return undefined;  
-    }  
-  
-    keys(): K[] {  
-      return [];  
-    }  
-  }

package/lib/implements/threaddb/lsstoreds/cyclic_batch.ts

@@ -1,87 +0,0 @@
-import { Datastore,Batch ,Key} from 'interface-datastore';
-
-  
-  type CyclicBatchOptions = {  
-    opsPerBatch?: number;  
-  };  
-  
-  export class CyclicBatch implements Batch {  
-    private currentBatch: Batch | null = null;  
-    private pendingOps = 0;  
-    private readonly threshold: number;  
-    private closed = false;  
-  
-    constructor(  
-      private readonly ds: Datastore,  
-      options: CyclicBatchOptions = {}  
-    ) {  
-      this.threshold = options.opsPerBatch || 20;  
-    }  
-  
-    private async initializeBatch(): Promise<void> {  
-      if (!this.currentBatch && !this.closed) {  
-        this.currentBatch =  this.ds.batch();
-      }  
-    }  
-  
-    private async cycle(): Promise<void> {  
-      if (this.closed) {  
-        throw new Error('Cyclic batch is closed');  
-      }  
-  
-      if (this.pendingOps < this.threshold || !this.currentBatch) {  
-        return;  
-      }  
-  
-      try {  
-        await this.currentBatch.commit();  
-        this.pendingOps = 0;  
-        this.currentBatch =  this.ds.batch();
-      } catch (err:any) {  
-        throw new Error(`Batch operation failed: ${err.message}`);  
-      }  
-    }  
-  
-    async put(key: Key, value: Uint8Array): Promise<void> {  
-      if (this.closed) {  
-        throw new Error('Cannot write to closed batch');  
-      }  
-  
-      await this.initializeBatch();  
-      await this.cycle();  
-      
-      if (this.currentBatch) {  
-        this.pendingOps++;  
-        await this.currentBatch.put(key, value);  
-      }  
-    }  
-  
-    async delete(key: Key): Promise<void> {  
-      if (this.closed) {  
-        throw new Error('Cannot write to closed batch');  
-      }  
-  
-      await this.initializeBatch();  
-      await this.cycle();  
-  
-      if (this.currentBatch) {  
-        this.pendingOps++;  
-        await this.currentBatch.delete(key);  
-      }  
-    }  
-  
-    async commit(): Promise<void> {  
-      if (this.closed) {  
-        throw new Error('Batch already committed');  
-      }  
-  
-      if (this.currentBatch) {  
-        await this.currentBatch.commit();  
-        this.currentBatch = null;  
-      }  
-      
-      this.closed = true;  
-      this.pendingOps = 0;  
-    }  
-  }  
-