web-dc-api 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.js +1 -1
- package/dist/dc.min.js +1 -1
- package/dist/esm/index.js +1 -1
- package/dist/index.d.ts +934 -878
- package/package.json +4 -8
- package/dist/cjs/helia-core-B1Xqha7a.js +0 -1
- package/dist/cjs/helia-core-D8Uv1KjQ.js +0 -1
- package/dist/cjs/polkadot-api-7PhQf3ws.js +0 -1
- package/dist/cjs/polkadot-api-CtrJVWuZ.js +0 -1
- package/dist/esm/chunks/helia-core-BxMqyK2Y.js +0 -1
- package/dist/esm/chunks/helia-core-DMXRpcO-.js +0 -1
- package/dist/esm/chunks/polkadot-api-5Y9Bw8VT.js +0 -1
- package/dist/esm/chunks/polkadot-api-D69Ioun_.js +0 -1
- package/lib/common/blowfish/block.ts +0 -259
- package/lib/common/blowfish/cipher.ts +0 -144
- package/lib/common/blowfish/const.ts +0 -195
- package/lib/common/chain.ts +0 -469
- package/lib/common/commonclient.ts +0 -202
- package/lib/common/constants.ts +0 -55
- package/lib/common/dc-key/ed25519.ts +0 -343
- package/lib/common/dc-key/keyManager.ts +0 -424
- package/lib/common/dcapi.ts +0 -98
- package/lib/common/dcutil.ts +0 -627
- package/lib/common/define.ts +0 -70
- package/lib/common/error.ts +0 -67
- package/lib/common/grpc-dc.ts +0 -104
- package/lib/common/module-system.ts +0 -184
- package/lib/common/service-worker.ts +0 -234
- package/lib/common/types/types.ts +0 -344
- package/lib/dc.ts +0 -701
- package/lib/implements/account/client.ts +0 -185
- package/lib/implements/account/manager.ts +0 -683
- package/lib/implements/aiproxy/client.ts +0 -357
- package/lib/implements/aiproxy/manager.ts +0 -670
- package/lib/implements/cache/client.ts +0 -105
- package/lib/implements/cache/manager.ts +0 -127
- package/lib/implements/comment/client.ts +0 -982
- package/lib/implements/comment/manager.ts +0 -1151
- package/lib/implements/dc/client.ts +0 -51
- package/lib/implements/dc/manager.ts +0 -33
- package/lib/implements/file/client.ts +0 -253
- package/lib/implements/file/file-cache-manager.ts +0 -142
- package/lib/implements/file/manager.ts +0 -1240
- package/lib/implements/file/seekableFileStream.ts +0 -344
- package/lib/implements/file/streamwriter.ts +0 -322
- package/lib/implements/keyvalue/client.ts +0 -376
- package/lib/implements/keyvalue/manager.ts +0 -759
- package/lib/implements/message/client.ts +0 -250
- package/lib/implements/message/manager.ts +0 -215
- package/lib/implements/threaddb/cbor/coding.ts +0 -62
- package/lib/implements/threaddb/cbor/event.ts +0 -336
- package/lib/implements/threaddb/cbor/node.ts +0 -542
- package/lib/implements/threaddb/cbor/record.ts +0 -398
- package/lib/implements/threaddb/common/AsyncMutex.ts +0 -24
- package/lib/implements/threaddb/common/addrinfo.ts +0 -135
- package/lib/implements/threaddb/common/dispatcher.ts +0 -81
- package/lib/implements/threaddb/common/idbstore-adapter.ts +0 -260
- package/lib/implements/threaddb/common/json-patcher.ts +0 -204
- package/lib/implements/threaddb/common/key.ts +0 -290
- package/lib/implements/threaddb/common/level-adapter.ts +0 -235
- package/lib/implements/threaddb/common/lineReader.ts +0 -79
- package/lib/implements/threaddb/common/logstore.ts +0 -215
- package/lib/implements/threaddb/common/transformed-datastore.ts +0 -308
- package/lib/implements/threaddb/core/app.ts +0 -206
- package/lib/implements/threaddb/core/core.ts +0 -230
- package/lib/implements/threaddb/core/db.ts +0 -249
- package/lib/implements/threaddb/core/event.ts +0 -54
- package/lib/implements/threaddb/core/head.ts +0 -89
- package/lib/implements/threaddb/core/identity.ts +0 -171
- package/lib/implements/threaddb/core/logstore.ts +0 -137
- package/lib/implements/threaddb/core/options.ts +0 -14
- package/lib/implements/threaddb/core/record.ts +0 -54
- package/lib/implements/threaddb/db/collection.ts +0 -1910
- package/lib/implements/threaddb/db/db.ts +0 -698
- package/lib/implements/threaddb/db/json2Query.ts +0 -192
- package/lib/implements/threaddb/db/query.ts +0 -524
- package/lib/implements/threaddb/dbclient.ts +0 -543
- package/lib/implements/threaddb/dbmanager.ts +0 -1906
- package/lib/implements/threaddb/lsstoreds/addr_book.ts +0 -549
- package/lib/implements/threaddb/lsstoreds/cache.ts +0 -36
- package/lib/implements/threaddb/lsstoreds/cyclic_batch.ts +0 -87
- package/lib/implements/threaddb/lsstoreds/global.ts +0 -151
- package/lib/implements/threaddb/lsstoreds/headbook.ts +0 -373
- package/lib/implements/threaddb/lsstoreds/keybook.ts +0 -297
- package/lib/implements/threaddb/lsstoreds/logstore.ts +0 -29
- package/lib/implements/threaddb/lsstoreds/metadata.ts +0 -223
- package/lib/implements/threaddb/net/define.ts +0 -149
- package/lib/implements/threaddb/net/grpcClient.ts +0 -589
- package/lib/implements/threaddb/net/grpcserver.ts +0 -146
- package/lib/implements/threaddb/net/net.ts +0 -2047
- package/lib/implements/threaddb/pb/lstore.proto +0 -38
- package/lib/implements/threaddb/pb/lstore.ts +0 -393
- package/lib/implements/threaddb/pb/lstore_pb.d.ts +0 -433
- package/lib/implements/threaddb/pb/lstore_pb.js +0 -1085
- package/lib/implements/threaddb/pb/net.proto +0 -194
- package/lib/implements/threaddb/pb/net_pb.d.ts +0 -2349
- package/lib/implements/threaddb/pb/net_pb.js +0 -5525
- package/lib/implements/threaddb/pb/proto-custom-types.ts +0 -212
- package/lib/implements/util/client.ts +0 -72
- package/lib/implements/util/manager.ts +0 -146
- package/lib/implements/wallet/manager.ts +0 -671
- package/lib/index.ts +0 -57
- package/lib/interfaces/DCContext.ts +0 -51
- package/lib/interfaces/aiproxy-interface.ts +0 -145
- package/lib/interfaces/auth-interface.ts +0 -118
- package/lib/interfaces/cache-interface.ts +0 -22
- package/lib/interfaces/client-interface.ts +0 -11
- package/lib/interfaces/comment-interface.ts +0 -167
- package/lib/interfaces/components/news-component.ts +0 -0
- package/lib/interfaces/database-interface.ts +0 -169
- package/lib/interfaces/file-interface.ts +0 -120
- package/lib/interfaces/index.ts +0 -10
- package/lib/interfaces/keyvalue-interface.ts +0 -156
- package/lib/interfaces/message-interface.ts +0 -22
- package/lib/interfaces/util-interface.ts +0 -31
- package/lib/modules/aiproxy-module.ts +0 -246
- package/lib/modules/auth-module.ts +0 -753
- package/lib/modules/cache-module.ts +0 -99
- package/lib/modules/client-module.ts +0 -71
- package/lib/modules/comment-module.ts +0 -429
- package/lib/modules/components/news-components.ts +0 -390
- package/lib/modules/database-module.ts +0 -598
- package/lib/modules/file-module.ts +0 -291
- package/lib/modules/index.ts +0 -13
- package/lib/modules/keyvalue-module.ts +0 -379
- package/lib/modules/message-module.ts +0 -107
- package/lib/modules/util-module.ts +0 -148
- package/lib/polyfills/process-env-browser.ts +0 -1
- package/lib/proto/datasource.ts +0 -93
- package/lib/proto/dcnet.proto +0 -1601
- package/lib/proto/dcnet_proto.d.ts +0 -22857
- package/lib/proto/dcnet_proto.js +0 -55204
- package/lib/proto/dcnet_proto_sparse.js +0 -55166
- package/lib/proto/oidfetch.proto +0 -25
- package/lib/proto/oidfetch_proto.d.ts +0 -585
- package/lib/proto/oidfetch_proto.js +0 -1247
- package/lib/serverless/babel-browser.ts +0 -39
- package/lib/serverless/base_entity.ts +0 -78
- package/lib/serverless/base_repository.ts +0 -414
- package/lib/serverless/browser_schema_extractor.ts +0 -283
- package/lib/serverless/decorator_factory.ts +0 -322
- package/lib/util/BrowserLineReader.ts +0 -73
- package/lib/util/base64.ts +0 -105
- package/lib/util/bcrypt.ts +0 -206
- package/lib/util/curve25519Encryption.ts +0 -418
- package/lib/util/dccrypt.ts +0 -73
- package/lib/util/logger.ts +0 -104
- package/lib/util/utils.ts +0 -289
|
@@ -1,2047 +0,0 @@
|
|
|
1
|
-
import crypto from "crypto";
|
|
2
|
-
|
|
3
|
-
import {
|
|
4
|
-
peerIdFromPublicKey,
|
|
5
|
-
peerIdFromPrivateKey,
|
|
6
|
-
peerIdFromMultihash,
|
|
7
|
-
peerIdFromString,
|
|
8
|
-
} from "@libp2p/peer-id";
|
|
9
|
-
import { keys } from "@libp2p/crypto";
|
|
10
|
-
import {
|
|
11
|
-
Multiaddr as TMultiaddr,
|
|
12
|
-
multiaddr,
|
|
13
|
-
} from "@multiformats/multiaddr"; // 多地址库
|
|
14
|
-
import { Head } from "../core/head";
|
|
15
|
-
import { ThreadID } from "@textile/threads-id";
|
|
16
|
-
import { Ed25519PrivKey, Ed25519PubKey } from "../../../common/dc-key/ed25519";
|
|
17
|
-
import type {
|
|
18
|
-
PeerId,
|
|
19
|
-
PublicKey,
|
|
20
|
-
PrivateKey,
|
|
21
|
-
Ed25519PublicKey,
|
|
22
|
-
} from "@libp2p/interface";
|
|
23
|
-
import { SymmetricKey, Key as ThreadKey } from "../common/key";
|
|
24
|
-
import { validateIDData } from "../lsstoreds/global";
|
|
25
|
-
import {
|
|
26
|
-
ThreadInfo,
|
|
27
|
-
IThreadLogInfo,
|
|
28
|
-
SymKey,
|
|
29
|
-
IThreadInfo,
|
|
30
|
-
ThreadMuliaddr,
|
|
31
|
-
} from "../core/core";
|
|
32
|
-
import { ThreadToken } from "../core/identity";
|
|
33
|
-
import { ILogstore } from "../core/logstore";
|
|
34
|
-
import { Datastore } from "interface-datastore";
|
|
35
|
-
import { Blocks } from "helia";
|
|
36
|
-
import { DAGCBOR } from "@helia/dag-cbor"; // DAGService
|
|
37
|
-
import * as dagCBOR from "@ipld/dag-cbor";
|
|
38
|
-
import { IRecord, IThreadRecord } from "../core/record";
|
|
39
|
-
import { DCGrpcServer } from "./grpcserver";
|
|
40
|
-
import { Libp2p } from "@libp2p/interface";
|
|
41
|
-
import { dc_protocol } from "../../../common/define";
|
|
42
|
-
import { EventFromNode, Event } from "../cbor/event";
|
|
43
|
-
import { Node } from "../cbor/node";
|
|
44
|
-
import { IPLDNode } from "../core/core";
|
|
45
|
-
import {
|
|
46
|
-
ThreadRecord,
|
|
47
|
-
TimestampedRecord,
|
|
48
|
-
PeerRecords,
|
|
49
|
-
netPullingLimit,
|
|
50
|
-
} from "./define";
|
|
51
|
-
import { CID } from "multiformats/cid";
|
|
52
|
-
import { getHeadUndef } from "../core/head";
|
|
53
|
-
import {
|
|
54
|
-
GetRecord,
|
|
55
|
-
CreateRecord,
|
|
56
|
-
logToProto,
|
|
57
|
-
logFromProto,
|
|
58
|
-
CreateRecordConfig,
|
|
59
|
-
} from "../cbor/record";
|
|
60
|
-
import { IThreadEvent } from "../core/event";
|
|
61
|
-
import { DBGrpcClient } from "./grpcClient";
|
|
62
|
-
import { DBClient } from "../dbclient";
|
|
63
|
-
import { Client } from "../../../common/dcapi";
|
|
64
|
-
import { App, Connector, Net, PubKey, Token } from "../core/app";
|
|
65
|
-
import { ChainUtil } from "../../../common/chain";
|
|
66
|
-
import { BrowserType, DcUtil } from "../../../common/dcutil";
|
|
67
|
-
import { CreateEvent } from "../cbor/event";
|
|
68
|
-
import { Errors } from "../core/db";
|
|
69
|
-
import { net as net_pb } from "../pb/net_pb";
|
|
70
|
-
import { PermanentAddrTTL } from "../common/logstore";
|
|
71
|
-
import { PeerStatus } from "../../../common/types/types";
|
|
72
|
-
import {
|
|
73
|
-
PeerIDConverter,
|
|
74
|
-
MultiaddrConverter,
|
|
75
|
-
CidConverter,
|
|
76
|
-
ThreadIDConverter,
|
|
77
|
-
KeyConverter,
|
|
78
|
-
ProtoKeyConverter,
|
|
79
|
-
json,
|
|
80
|
-
} from "../pb/proto-custom-types";
|
|
81
|
-
import * as buffer from "buffer/";
|
|
82
|
-
import { AsyncMutex } from "../common/AsyncMutex";
|
|
83
|
-
import { DCContext } from "../../../interfaces";
|
|
84
|
-
import { time } from "console";
|
|
85
|
-
const { Buffer } = buffer;
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* Creates a new ThreadRecord
|
|
89
|
-
*/
|
|
90
|
-
function newRecord(r: IRecord, id: ThreadID, lid: PeerId): IThreadRecord {
|
|
91
|
-
return new ThreadRecord(r, id, lid);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// 定义 Network 类
|
|
95
|
-
export class Network implements Net {
|
|
96
|
-
bstore: Blocks;
|
|
97
|
-
private logstore: ILogstore;
|
|
98
|
-
private dcChain: ChainUtil;
|
|
99
|
-
private dc: DcUtil;
|
|
100
|
-
private dagService: DAGCBOR;
|
|
101
|
-
private hostID: string;
|
|
102
|
-
private context: DCContext;
|
|
103
|
-
private server: DCGrpcServer;
|
|
104
|
-
private libp2p: Libp2p;
|
|
105
|
-
private connectors: Record<string, Connector>;
|
|
106
|
-
private cachePeers: Record<string, TMultiaddr> = {};
|
|
107
|
-
private threadMutexes: Record<string, AsyncMutex> = {};
|
|
108
|
-
private pushQueue: Array<{ tid: ThreadID; lid: PeerId; rec: IRecord; counter: number }> = [];
|
|
109
|
-
private pushWorkerStarted = false;
|
|
110
|
-
|
|
111
|
-
constructor(
|
|
112
|
-
dcUtil: DcUtil,
|
|
113
|
-
dcChain: ChainUtil,
|
|
114
|
-
libp2p: Libp2p,
|
|
115
|
-
grpcServer: DCGrpcServer,
|
|
116
|
-
logstore: ILogstore,
|
|
117
|
-
bstore: Blocks,
|
|
118
|
-
dagService: DAGCBOR,
|
|
119
|
-
context: DCContext
|
|
120
|
-
) {
|
|
121
|
-
this.logstore = logstore;
|
|
122
|
-
this.hostID = libp2p.peerId.toString();
|
|
123
|
-
this.context = context;
|
|
124
|
-
this.bstore = bstore;
|
|
125
|
-
this.dagService = dagService;
|
|
126
|
-
this.libp2p = libp2p;
|
|
127
|
-
this.server = grpcServer;
|
|
128
|
-
this.connectors = {};
|
|
129
|
-
this.dcChain = dcChain;
|
|
130
|
-
this.dc = dcUtil;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
// 签名,后续应该改成发送到钱包iframe中签名,发送数据包含payload和用户公钥
|
|
134
|
-
sign = async (payload: Uint8Array): Promise<Uint8Array> => {
|
|
135
|
-
if (!this.context) {
|
|
136
|
-
throw new Error("privKey is null");
|
|
137
|
-
}
|
|
138
|
-
const signature = this.context.sign(payload);
|
|
139
|
-
return signature;
|
|
140
|
-
};
|
|
141
|
-
|
|
142
|
-
getDagService(): DAGCBOR {
|
|
143
|
-
return this.dagService;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
async getClient(peerId: PeerId): Promise<[Client | null, Error | null]> {
|
|
147
|
-
try {
|
|
148
|
-
let cachedFlag = true;
|
|
149
|
-
const cacheAddr = this.cachePeers[peerId.toString()];
|
|
150
|
-
let peerAddr: TMultiaddr | null = cacheAddr || null;
|
|
151
|
-
let peerStatus: PeerStatus | null = PeerStatus.PeerStatusOnline;
|
|
152
|
-
if (!cacheAddr) {
|
|
153
|
-
cachedFlag = false;
|
|
154
|
-
[peerAddr, peerStatus] = await this.dcChain.getDcNodeWebrtcDirectAddr(
|
|
155
|
-
peerId.toString()
|
|
156
|
-
);
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
if (!peerAddr) {
|
|
160
|
-
throw new Error("peerAddr is null");
|
|
161
|
-
}
|
|
162
|
-
if (peerStatus !== PeerStatus.PeerStatusOnline) {
|
|
163
|
-
throw new Error("peerStatus is not online");
|
|
164
|
-
}
|
|
165
|
-
if (!this.context.publicKey) {
|
|
166
|
-
throw new Error("publicKey is null");
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
const addr = multiaddr(peerAddr);
|
|
170
|
-
const client = new Client(this.libp2p, this.bstore, addr, dc_protocol);
|
|
171
|
-
//获取token
|
|
172
|
-
const token = await client.GetToken(
|
|
173
|
-
this.context.appInfo.appId || "",
|
|
174
|
-
this.context.publicKey.string(),
|
|
175
|
-
(payload: Uint8Array): Promise<Uint8Array> => {
|
|
176
|
-
return this.sign(payload);
|
|
177
|
-
}
|
|
178
|
-
);
|
|
179
|
-
if (!token) {
|
|
180
|
-
if (cachedFlag) {
|
|
181
|
-
let _: PeerStatus | null = null;
|
|
182
|
-
[peerAddr, _] = await this.dcChain.getDcNodeWebrtcDirectAddr(
|
|
183
|
-
peerId.toString()
|
|
184
|
-
);
|
|
185
|
-
delete this.cachePeers[peerId.toString()];
|
|
186
|
-
if (!peerAddr) {
|
|
187
|
-
throw new Error("peerAddr is null");
|
|
188
|
-
}
|
|
189
|
-
const addr = multiaddr(peerAddr);
|
|
190
|
-
const client = new Client(
|
|
191
|
-
this.libp2p,
|
|
192
|
-
this.bstore,
|
|
193
|
-
addr,
|
|
194
|
-
dc_protocol
|
|
195
|
-
);
|
|
196
|
-
const token = await client.GetToken(
|
|
197
|
-
this.context.appInfo.appId || "",
|
|
198
|
-
this.context.publicKey.string(),
|
|
199
|
-
(payload: Uint8Array) => {
|
|
200
|
-
return this.sign(payload);
|
|
201
|
-
}
|
|
202
|
-
);
|
|
203
|
-
if (token) {
|
|
204
|
-
this.cachePeers[peerId.toString()] = peerAddr;
|
|
205
|
-
return [client, null];
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
throw new Error("get token is null");
|
|
209
|
-
}
|
|
210
|
-
this.cachePeers[peerId.toString()] = peerAddr;
|
|
211
|
-
return [client, null];
|
|
212
|
-
} catch (err) {
|
|
213
|
-
return [null, err as Error];
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
getMutexForThread(threadId: string): AsyncMutex {
|
|
218
|
-
if (!this.threadMutexes[threadId]) {
|
|
219
|
-
this.threadMutexes[threadId] = new AsyncMutex();
|
|
220
|
-
}
|
|
221
|
-
return this.threadMutexes[threadId];
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
async getPeers(id: ThreadID): Promise<PeerId[] | undefined> {
|
|
225
|
-
const peers = await this.dcChain.getObjNodes(id.toString());
|
|
226
|
-
if (!peers) {
|
|
227
|
-
return undefined;
|
|
228
|
-
}
|
|
229
|
-
const peerIds: PeerId[] = [];
|
|
230
|
-
for (const peer of peers) {
|
|
231
|
-
const peerStr = Buffer.from(peer.slice(2), "hex").toString("utf8");
|
|
232
|
-
const peerId = peerIdFromString(peerStr);
|
|
233
|
-
if (!peerId) {
|
|
234
|
-
continue;
|
|
235
|
-
}
|
|
236
|
-
peerIds.push(peerId);
|
|
237
|
-
}
|
|
238
|
-
return peerIds;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
/**
|
|
242
|
-
* 创建threaddb
|
|
243
|
-
*/
|
|
244
|
-
async createThread(
|
|
245
|
-
id: ThreadID,
|
|
246
|
-
options: {
|
|
247
|
-
token: ThreadToken;
|
|
248
|
-
logKey?: Ed25519PrivKey | Ed25519PubKey;
|
|
249
|
-
threadKey?: ThreadKey;
|
|
250
|
-
}
|
|
251
|
-
): Promise<ThreadInfo> {
|
|
252
|
-
if (!this.context.publicKey) {
|
|
253
|
-
throw new Error("Identity creation failed.");
|
|
254
|
-
}
|
|
255
|
-
const identity = this.context.publicKey;
|
|
256
|
-
|
|
257
|
-
await this.ensureUniqueLog(id, options.logKey, identity);
|
|
258
|
-
const threadKey = options.threadKey || this.generateRandomKey();
|
|
259
|
-
|
|
260
|
-
const threadInfo = new ThreadInfo(id, [], [], threadKey);
|
|
261
|
-
|
|
262
|
-
await this.logstore.addThread(threadInfo);
|
|
263
|
-
const logInfo = await this.createLog(id, options.logKey, identity);
|
|
264
|
-
threadInfo.logs.push(logInfo);
|
|
265
|
-
return threadInfo;
|
|
266
|
-
}
|
|
267
|
-
|
|
268
|
-
/**
|
|
269
|
-
* 从多地址添加threaddb
|
|
270
|
-
*
|
|
271
|
-
* @param addr 包含threaddb ID的多地址
|
|
272
|
-
* @param options threaddb 选项
|
|
273
|
-
* @returns 带有地址的threaddb 信息
|
|
274
|
-
*/
|
|
275
|
-
async addThread(
|
|
276
|
-
addr: ThreadMuliaddr,
|
|
277
|
-
options: {
|
|
278
|
-
token?: ThreadToken | undefined;
|
|
279
|
-
logKey?: Ed25519PrivKey | Ed25519PubKey | undefined;
|
|
280
|
-
threadKey?: ThreadKey | undefined;
|
|
281
|
-
} = {}
|
|
282
|
-
): Promise<ThreadInfo> {
|
|
283
|
-
try {
|
|
284
|
-
// 从多地址提取threaddb ID
|
|
285
|
-
const idStr = addr.id.toString();
|
|
286
|
-
if (!idStr) {
|
|
287
|
-
throw new Error("Invalid thread address");
|
|
288
|
-
}
|
|
289
|
-
const id = ThreadID.fromString(idStr);
|
|
290
|
-
// 验证身份
|
|
291
|
-
let identity = await this.validate(id, options.token);
|
|
292
|
-
if (identity) {
|
|
293
|
-
console.debug(`Adding thread with identity: ${identity.toString()}`);
|
|
294
|
-
} else {
|
|
295
|
-
identity = this.context.publicKey;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
// 确保日志唯一性
|
|
299
|
-
await this.ensureUniqueLog(id, options.logKey, identity);
|
|
300
|
-
|
|
301
|
-
// 分离threaddb 组件以获取对等点地址
|
|
302
|
-
// const threadComp = `/thread/${id.toString()}`;
|
|
303
|
-
// const peerAddr = multiaddr(addr.toString().split(threadComp)[0]);
|
|
304
|
-
const peerAddr = addr.addr;
|
|
305
|
-
// 获取对等点信息
|
|
306
|
-
const peerId = peerAddr.getPeerId();
|
|
307
|
-
if (!peerId) {
|
|
308
|
-
throw new Error("Invalid peer address");
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
const pid = peerIdFromString(peerId);
|
|
312
|
-
const addFromSelf = pid.toString() === this.hostID;
|
|
313
|
-
|
|
314
|
-
// 如果我们从自己添加,检查threaddb 是否存在
|
|
315
|
-
if (addFromSelf) {
|
|
316
|
-
try {
|
|
317
|
-
await this.logstore.getThread(id);
|
|
318
|
-
} catch (err: any) {
|
|
319
|
-
if (err.message === "Thread not found") {
|
|
320
|
-
throw new Error(`Cannot retrieve thread from self: ${err.message}`);
|
|
321
|
-
}
|
|
322
|
-
throw err;
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
// 添加threaddb 到存储
|
|
327
|
-
const threadInfo = new ThreadInfo(id, [], [], options.threadKey);
|
|
328
|
-
await this.logstore.addThread(threadInfo);
|
|
329
|
-
|
|
330
|
-
// 如果可以读取或有日志密钥,则创建日志
|
|
331
|
-
if (
|
|
332
|
-
(options.threadKey && options.threadKey.canRead()) ||
|
|
333
|
-
options.logKey
|
|
334
|
-
) {
|
|
335
|
-
const logInfo = await this.createLog(id, options.logKey, identity);
|
|
336
|
-
threadInfo.logs.push(logInfo);
|
|
337
|
-
}
|
|
338
|
-
// 如果不是从自己添加,则连接并获取日志
|
|
339
|
-
if (!addFromSelf) {
|
|
340
|
-
// 从对等点更新日志
|
|
341
|
-
await this.updateLogsFromPeer(id, pid);
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
// 返回带有地址的threaddb 信息
|
|
345
|
-
return this.getThreadWithAddrs(id);
|
|
346
|
-
} catch (err) {
|
|
347
|
-
throw new Error(
|
|
348
|
-
`Failed to add thread: ${
|
|
349
|
-
err instanceof Error ? err.message : String(err)
|
|
350
|
-
}`
|
|
351
|
-
);
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
/**
|
|
356
|
-
* 获取threaddb 信息
|
|
357
|
-
* 返回包含地址的threaddb 信息对象
|
|
358
|
-
*
|
|
359
|
-
* @param id threaddb ID
|
|
360
|
-
* @param options 选项,可以包含令牌等
|
|
361
|
-
* @returns 包含地址的threaddb 信息
|
|
362
|
-
* @throws 如果threaddb 验证失败或threaddb 不存在
|
|
363
|
-
*/
|
|
364
|
-
async getThread(
|
|
365
|
-
id: ThreadID,
|
|
366
|
-
options: { token?: ThreadToken } = {}
|
|
367
|
-
): Promise<ThreadInfo> {
|
|
368
|
-
try {
|
|
369
|
-
// 验证threaddb ID和令牌
|
|
370
|
-
await this.validate(id, options.token);
|
|
371
|
-
|
|
372
|
-
// 获取带有地址的threaddb 信息
|
|
373
|
-
return this.getThreadWithAddrs(id);
|
|
374
|
-
} catch (err) {
|
|
375
|
-
throw new Error(
|
|
376
|
-
`Error getting thread ${id.toString()}: ${
|
|
377
|
-
err instanceof Error ? err.message : String(err)
|
|
378
|
-
}`
|
|
379
|
-
);
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
/**
|
|
384
|
-
* 从连接的远程对等点获取线程信息
|
|
385
|
-
*
|
|
386
|
-
* @param id 线程ID
|
|
387
|
-
* @param options 线程选项
|
|
388
|
-
* @returns 线程信息对象
|
|
389
|
-
* @throws 如果连接刷新失败或获取线程信息失败
|
|
390
|
-
*/
|
|
391
|
-
async getThreadFromPeer(
|
|
392
|
-
id: ThreadID,
|
|
393
|
-
peerId: PeerId,
|
|
394
|
-
options: { token?: ThreadToken } = {}
|
|
395
|
-
): Promise<ThreadInfo> {
|
|
396
|
-
try {
|
|
397
|
-
const [client, _] = await this.getClient(peerId);
|
|
398
|
-
if (!client) {
|
|
399
|
-
throw new Error("Failed to get client");
|
|
400
|
-
}
|
|
401
|
-
const dbClient = new DBClient(client, this.dc, this, this.logstore);
|
|
402
|
-
|
|
403
|
-
// 发送请求
|
|
404
|
-
try {
|
|
405
|
-
const threadInfo = dbClient.getThreadFromPeer(id, peerId, options);
|
|
406
|
-
return threadInfo;
|
|
407
|
-
} catch (err) {
|
|
408
|
-
throw new Error(
|
|
409
|
-
`Error getting thread from peer: ${
|
|
410
|
-
err instanceof Error ? err.message : String(err)
|
|
411
|
-
}`
|
|
412
|
-
);
|
|
413
|
-
}
|
|
414
|
-
} catch (err) {
|
|
415
|
-
throw new Error(
|
|
416
|
-
`Error getting thread from peer: ${
|
|
417
|
-
err instanceof Error ? err.message : String(err)
|
|
418
|
-
}`
|
|
419
|
-
);
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
/**
|
|
424
|
-
* 删除线程
|
|
425
|
-
*
|
|
426
|
-
* @param id 线程ID
|
|
427
|
-
* @param options 线程选项
|
|
428
|
-
* @returns 无返回值
|
|
429
|
-
* @throws 如果验证失败或线程正在被使用
|
|
430
|
-
*/
|
|
431
|
-
async deleteThread(
|
|
432
|
-
id: ThreadID,
|
|
433
|
-
options: { token?: ThreadToken; apiToken?: Token } = {}
|
|
434
|
-
): Promise<void> {
|
|
435
|
-
try {
|
|
436
|
-
// 验证线程ID和令牌
|
|
437
|
-
await this.validate(id, options.token);
|
|
438
|
-
|
|
439
|
-
// 检查线程是否被应用使用
|
|
440
|
-
const [_, ok] = this.getConnectorProtected(id, options.apiToken);
|
|
441
|
-
if (!ok) {
|
|
442
|
-
throw new Error("Cannot delete thread: thread in use");
|
|
443
|
-
}
|
|
444
|
-
console.debug(`Deleting thread ${id.toString()}...`);
|
|
445
|
-
const mutex = this.getMutexForThread(id.toString());
|
|
446
|
-
await mutex.acquire();
|
|
447
|
-
try {
|
|
448
|
-
// 执行删除操作
|
|
449
|
-
await this.logstore.deleteThread(id);
|
|
450
|
-
delete this.connectors[id.toString()];
|
|
451
|
-
} finally {
|
|
452
|
-
mutex.release();
|
|
453
|
-
}
|
|
454
|
-
} catch (err) {
|
|
455
|
-
throw new Error(
|
|
456
|
-
`Failed to delete thread: ${
|
|
457
|
-
err instanceof Error ? err.message : String(err)
|
|
458
|
-
}`
|
|
459
|
-
);
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
/**
|
|
464
|
-
* 验证threaddb ID 和 Token
|
|
465
|
-
*/
|
|
466
|
-
async validate(
|
|
467
|
-
id: ThreadID,
|
|
468
|
-
token?: ThreadToken
|
|
469
|
-
): Promise<Ed25519PubKey | undefined> {
|
|
470
|
-
if (!validateIDData(id.toBytes())) {
|
|
471
|
-
throw new Error("Invalid thread ID.");
|
|
472
|
-
}
|
|
473
|
-
if (!token) {
|
|
474
|
-
return;
|
|
475
|
-
}
|
|
476
|
-
return token.pubKey();
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
/**
|
|
480
|
-
* 确保日志唯一性
|
|
481
|
-
* 检查给定threaddb 是否已存在具有相同密钥或身份的日志
|
|
482
|
-
*/
|
|
483
|
-
async ensureUniqueLog(
|
|
484
|
-
id: ThreadID,
|
|
485
|
-
key?: Ed25519PrivKey | Ed25519PubKey,
|
|
486
|
-
identity?: PublicKey
|
|
487
|
-
): Promise<void> {
|
|
488
|
-
try {
|
|
489
|
-
const thrd = await this.logstore.getThread(id);
|
|
490
|
-
|
|
491
|
-
// threaddb 存在,继续检查
|
|
492
|
-
let lid: PeerId;
|
|
493
|
-
|
|
494
|
-
if (key) {
|
|
495
|
-
// 根据密钥类型处理
|
|
496
|
-
if (key instanceof Ed25519PubKey) {
|
|
497
|
-
lid = peerIdFromPublicKey(key);
|
|
498
|
-
} else if (key instanceof Ed25519PrivKey) {
|
|
499
|
-
lid = peerIdFromPrivateKey(key);
|
|
500
|
-
} else {
|
|
501
|
-
throw new Error("Invalid log key");
|
|
502
|
-
}
|
|
503
|
-
} else {
|
|
504
|
-
// 没有提供密钥,检查是否有此身份的日志
|
|
505
|
-
if (!identity) {
|
|
506
|
-
throw new Error("Either key or identity must be provided");
|
|
507
|
-
}
|
|
508
|
-
|
|
509
|
-
try {
|
|
510
|
-
const lidb = await this.logstore.metadata.getBytes(
|
|
511
|
-
id,
|
|
512
|
-
identity.toString()
|
|
513
|
-
);
|
|
514
|
-
if (!lidb || lidb.length === 0) {
|
|
515
|
-
// 检查是否有旧式"own"(未索引)日志
|
|
516
|
-
if (identity.equals(this.context.publicKey)) {
|
|
517
|
-
const firstPrivKeyLog = thrd.getFirstPrivKeyLog();
|
|
518
|
-
if (firstPrivKeyLog && firstPrivKeyLog.privKey) {
|
|
519
|
-
throw new Error("Thread exists");
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
return;
|
|
523
|
-
}
|
|
524
|
-
const lidbstr = new TextDecoder().decode(lidb);
|
|
525
|
-
// 从字节转换为PeerId
|
|
526
|
-
lid = peerIdFromString(lidbstr);
|
|
527
|
-
} catch (error) {
|
|
528
|
-
throw error;
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
try {
|
|
533
|
-
const lginfo = await this.logstore.getLog(id, lid);
|
|
534
|
-
if (lginfo) {
|
|
535
|
-
// 如果到达这里,说明日志存在
|
|
536
|
-
throw new Error("Log exists");
|
|
537
|
-
}
|
|
538
|
-
} catch (error: any) {
|
|
539
|
-
if (error.message === Errors.ErrLogNotFound.message) {
|
|
540
|
-
return;
|
|
541
|
-
}
|
|
542
|
-
throw error;
|
|
543
|
-
}
|
|
544
|
-
} catch (error: any) {
|
|
545
|
-
if (error.message === Errors.ErrThreadNotFound.message) {
|
|
546
|
-
return;
|
|
547
|
-
}
|
|
548
|
-
throw error;
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
/**
|
|
553
|
-
* Get thread information with addresses
|
|
554
|
-
*/
|
|
555
|
-
async getThreadWithAddrs(id: ThreadID): Promise<ThreadInfo> {
|
|
556
|
-
try {
|
|
557
|
-
const tinfo = await this.logstore.getThread(id);
|
|
558
|
-
// Get host addresses
|
|
559
|
-
const hostAddrs = this.libp2p.getMultiaddrs();
|
|
560
|
-
const resultAddrs: ThreadMuliaddr[] = [];
|
|
561
|
-
|
|
562
|
-
// Encapsulate each address with peer and thread components
|
|
563
|
-
for (const addr of hostAddrs) {
|
|
564
|
-
const withPeerId = addr.encapsulate(
|
|
565
|
-
`/p2p/${this.libp2p.peerId.toString()}`
|
|
566
|
-
);
|
|
567
|
-
const threadMultiaddr = new ThreadMuliaddr(withPeerId, tinfo.id);
|
|
568
|
-
resultAddrs.push(threadMultiaddr);
|
|
569
|
-
}
|
|
570
|
-
tinfo.addrs = resultAddrs;
|
|
571
|
-
return tinfo;
|
|
572
|
-
} catch (err) {
|
|
573
|
-
throw err;
|
|
574
|
-
}
|
|
575
|
-
}
|
|
576
|
-
|
|
577
|
-
/**
|
|
578
|
-
* Pull thread updates from peers
|
|
579
|
-
*/
|
|
580
|
-
async pullThread(id: ThreadID, timeout: number, options: { token?: ThreadToken | undefined; multiPeersFlag?: boolean | undefined;}): Promise<void> {
|
|
581
|
-
try {
|
|
582
|
-
let recs: Record<string, PeerRecords> = {};
|
|
583
|
-
const mutex = this.getMutexForThread(id.toString());
|
|
584
|
-
await mutex.acquire();
|
|
585
|
-
try {
|
|
586
|
-
if (options.multiPeersFlag) {
|
|
587
|
-
recs = await this.pullThreadDeal(id, options.multiPeersFlag );
|
|
588
|
-
}else{
|
|
589
|
-
recs = await this.pullThreadDeal(id, false );
|
|
590
|
-
}
|
|
591
|
-
} finally {
|
|
592
|
-
mutex.release();
|
|
593
|
-
}
|
|
594
|
-
|
|
595
|
-
const [connector, appConnected] = this.getConnector(id);
|
|
596
|
-
|
|
597
|
-
// Handle records
|
|
598
|
-
const tRecords: TimestampedRecord[] = [];
|
|
599
|
-
|
|
600
|
-
for (const [lidStr, rec] of Object.entries(recs)) {
|
|
601
|
-
const lid = peerIdFromString(lidStr);
|
|
602
|
-
const rs = rec as PeerRecords;
|
|
603
|
-
if (appConnected) {
|
|
604
|
-
// 使用并发控制,但不分批 - 避免慢任务拖累整批
|
|
605
|
-
const maxConcurrency = 10;
|
|
606
|
-
let activePromises: Promise<void>[] = [];
|
|
607
|
-
let processedCount = 0;
|
|
608
|
-
|
|
609
|
-
const processRecord = async (r: any, index: number): Promise<void> => {
|
|
610
|
-
try {
|
|
611
|
-
const block = await r.getBlock(this.bstore);
|
|
612
|
-
const event =
|
|
613
|
-
block instanceof Event
|
|
614
|
-
? block
|
|
615
|
-
: await EventFromNode(block as Node);
|
|
616
|
-
|
|
617
|
-
const header = await event.getHeader(this.bstore);
|
|
618
|
-
const body = await event.getBody(this.bstore);
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
// Store internal blocks locally
|
|
622
|
-
await this.addMany([event, header, body]);
|
|
623
|
-
|
|
624
|
-
} catch (err) {
|
|
625
|
-
console.error(`预加载记录 ${index + 1} 失败:`, err);
|
|
626
|
-
// 继续处理其他记录
|
|
627
|
-
}
|
|
628
|
-
};
|
|
629
|
-
|
|
630
|
-
// 处理所有记录,但控制并发数
|
|
631
|
-
for (let i = 0; i < rs.records.length; i++) {
|
|
632
|
-
const r = rs.records[i]!;
|
|
633
|
-
// 创建处理Promise
|
|
634
|
-
const promise = processRecord(r, i).finally(() => {
|
|
635
|
-
processedCount++;
|
|
636
|
-
// 从活跃Promise列表中移除
|
|
637
|
-
const index = activePromises.indexOf(promise);
|
|
638
|
-
if (index > -1) {
|
|
639
|
-
activePromises.splice(index, 1);
|
|
640
|
-
}
|
|
641
|
-
});
|
|
642
|
-
|
|
643
|
-
activePromises.push(promise);
|
|
644
|
-
|
|
645
|
-
// 当达到最大并发数时,等待最快完成的一个
|
|
646
|
-
if (activePromises.length >= maxConcurrency) {
|
|
647
|
-
await Promise.race(activePromises);
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
// 等待所有剩余的Promise完成
|
|
651
|
-
await Promise.all(activePromises);
|
|
652
|
-
|
|
653
|
-
console.log(`所有 ${rs.records.length} 条记录预加载完成`);
|
|
654
|
-
//开始正式处理,前面数据已经拉取到本地,这时处理速度很快
|
|
655
|
-
let indexCounter = rs.counter - rs.records.length + 1;
|
|
656
|
-
|
|
657
|
-
for (let i = 0; i < rs.records.length; i++) {
|
|
658
|
-
const r = rs.records[i]!;
|
|
659
|
-
|
|
660
|
-
// Get blocks for validation
|
|
661
|
-
const block = await r.getBlock(this.bstore);
|
|
662
|
-
const event =
|
|
663
|
-
block instanceof Event
|
|
664
|
-
? block
|
|
665
|
-
: await EventFromNode(block as Node);
|
|
666
|
-
|
|
667
|
-
const header = await event.getHeader(this.bstore);
|
|
668
|
-
const body = await event.getBody(this.bstore);
|
|
669
|
-
|
|
670
|
-
// Store internal blocks locally
|
|
671
|
-
await this.addMany([event, header, body]);
|
|
672
|
-
|
|
673
|
-
const tRecord = newRecord(r, id, lid);
|
|
674
|
-
const counter = indexCounter + i;
|
|
675
|
-
const createtime = await connector!.getNetRecordCreateTime(tRecord);
|
|
676
|
-
|
|
677
|
-
tRecords.push({
|
|
678
|
-
record: r,
|
|
679
|
-
counter: counter,
|
|
680
|
-
createtime: createtime,
|
|
681
|
-
logid: lid,
|
|
682
|
-
});
|
|
683
|
-
}
|
|
684
|
-
} else {
|
|
685
|
-
await this.putRecords(id, lid, rs.records, rs.counter);
|
|
686
|
-
}
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
tRecords.sort((a, b) => {
|
|
690
|
-
if (a.createtime < b.createtime) return -1;
|
|
691
|
-
if (a.createtime > b.createtime) return 1;
|
|
692
|
-
return 0;
|
|
693
|
-
});
|
|
694
|
-
let i = 0;
|
|
695
|
-
|
|
696
|
-
// Process each record in order
|
|
697
|
-
for (const r of tRecords) {
|
|
698
|
-
await this.putRecords(id, r.logid, [r.record], r.counter);
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
} catch (err) {
|
|
702
|
-
throw err;
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
async addMany(nodes: IPLDNode[]): Promise<void> {
|
|
707
|
-
for (const node of nodes) {
|
|
708
|
-
await this.bstore.put(node.cid(), node.data());
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
|
-
/**
|
|
712
|
-
* Pull thread records from peers
|
|
713
|
-
*/
|
|
714
|
-
async pullThreadDeal(tid: ThreadID,multiPeersFlag: boolean=false): Promise<Record<string, PeerRecords>> {
|
|
715
|
-
try {
|
|
716
|
-
let [offsets, peers] = await this.threadOffsets(tid);
|
|
717
|
-
try {
|
|
718
|
-
const extPeers = await this.getPeers(tid);
|
|
719
|
-
if (extPeers && extPeers.length > 0) {
|
|
720
|
-
peers = extPeers;
|
|
721
|
-
}
|
|
722
|
-
} catch (err) {
|
|
723
|
-
console.error(
|
|
724
|
-
`Error getting peers for thread ${tid}: ${
|
|
725
|
-
err instanceof Error ? err.message : String(err)
|
|
726
|
-
}`
|
|
727
|
-
);
|
|
728
|
-
// Ignore getPeers errors
|
|
729
|
-
}
|
|
730
|
-
const pulledRecs: Record<string, PeerRecords> = {};
|
|
731
|
-
|
|
732
|
-
// Continue pulling until no more records
|
|
733
|
-
while (true) {
|
|
734
|
-
const recs = await this.getRecords(
|
|
735
|
-
peers,
|
|
736
|
-
tid,
|
|
737
|
-
offsets,
|
|
738
|
-
netPullingLimit,
|
|
739
|
-
multiPeersFlag
|
|
740
|
-
);
|
|
741
|
-
let continueFlag = false;
|
|
742
|
-
for (const [lidStr, rs] of Object.entries(recs)) {
|
|
743
|
-
if (rs.records.length > 0) {
|
|
744
|
-
const existing = pulledRecs[lidStr] || { records: [], counter: 0 };
|
|
745
|
-
const records = [...existing.records, ...rs.records];
|
|
746
|
-
|
|
747
|
-
pulledRecs[lidStr] = {
|
|
748
|
-
records: records,
|
|
749
|
-
counter: rs.counter,
|
|
750
|
-
};
|
|
751
|
-
|
|
752
|
-
const lastRecord = rs.records[rs.records.length - 1]!;
|
|
753
|
-
offsets[lidStr] = {
|
|
754
|
-
id: lastRecord.cid(),
|
|
755
|
-
counter: rs.counter,
|
|
756
|
-
};
|
|
757
|
-
|
|
758
|
-
if (rs.records.length >= netPullingLimit) {
|
|
759
|
-
continueFlag = true;
|
|
760
|
-
}
|
|
761
|
-
}
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
if (!continueFlag) {
|
|
765
|
-
break;
|
|
766
|
-
}
|
|
767
|
-
}
|
|
768
|
-
return pulledRecs;
|
|
769
|
-
} catch (err) {
|
|
770
|
-
throw err;
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
/**
|
|
775
|
-
* 创建日志,分配 privKey 和 pubKey
|
|
776
|
-
*/
|
|
777
|
-
async createLog(
|
|
778
|
-
id: ThreadID,
|
|
779
|
-
key?: Ed25519PrivKey | Ed25519PubKey,
|
|
780
|
-
identity?: PublicKey
|
|
781
|
-
): Promise<IThreadLogInfo> {
|
|
782
|
-
let privKey: PrivateKey | undefined;
|
|
783
|
-
let pubKey: PublicKey;
|
|
784
|
-
let peerId: PeerId;
|
|
785
|
-
if (!key) {
|
|
786
|
-
const keyPair = await keys.generateKeyPair("Ed25519");
|
|
787
|
-
privKey = new Ed25519PrivKey(keyPair.raw);
|
|
788
|
-
pubKey = keyPair.publicKey;
|
|
789
|
-
peerId = peerIdFromPrivateKey(privKey);
|
|
790
|
-
} else {
|
|
791
|
-
if (key instanceof Ed25519PrivKey) {
|
|
792
|
-
privKey = key;
|
|
793
|
-
pubKey = key.publicKey;
|
|
794
|
-
peerId = peerIdFromPrivateKey(key);
|
|
795
|
-
} else if (key instanceof Ed25519PubKey) {
|
|
796
|
-
pubKey = key;
|
|
797
|
-
peerId = peerIdFromPublicKey(key);
|
|
798
|
-
} else {
|
|
799
|
-
throw new Error("Invalid key type.");
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
const addr = multiaddr(`/p2p/${this.hostID}`); // 基于 hostID 生成地址
|
|
803
|
-
const head: Head = {
|
|
804
|
-
counter: 0,
|
|
805
|
-
};
|
|
806
|
-
const logInfo: IThreadLogInfo = {
|
|
807
|
-
privKey,
|
|
808
|
-
pubKey,
|
|
809
|
-
id: peerId,
|
|
810
|
-
addrs: [addr],
|
|
811
|
-
managed: true,
|
|
812
|
-
head: head,
|
|
813
|
-
} as IThreadLogInfo;
|
|
814
|
-
//标记本地log没有生成任何记录
|
|
815
|
-
await this.logstore.metadata.putString(
|
|
816
|
-
id,
|
|
817
|
-
"local_log_no_record_flag",
|
|
818
|
-
`${peerId.toString()}`
|
|
819
|
-
);
|
|
820
|
-
// 将日志添加到threaddb存储
|
|
821
|
-
await this.logstore.addLog(id, logInfo);
|
|
822
|
-
const logIDBytes = new TextEncoder().encode(peerId.toString());
|
|
823
|
-
await this.logstore.metadata.putBytes(
|
|
824
|
-
id,
|
|
825
|
-
identity?.toString() || "",
|
|
826
|
-
logIDBytes
|
|
827
|
-
);
|
|
828
|
-
return logInfo;
|
|
829
|
-
}
|
|
830
|
-
|
|
831
|
-
/**
|
|
832
|
-
* 生成随机threaddb 密钥
|
|
833
|
-
*/
|
|
834
|
-
generateRandomKey(): ThreadKey {
|
|
835
|
-
return new ThreadKey(SymmetricKey.new(), SymmetricKey.new());
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
/**
|
|
839
|
-
* Add connector for thread
|
|
840
|
-
*/
|
|
841
|
-
addConnector(id: ThreadID, conn: Connector): void {
|
|
842
|
-
this.connectors[id.toString()] = conn;
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
/**
|
|
846
|
-
* Get connector for thread
|
|
847
|
-
*/
|
|
848
|
-
getConnector(id: ThreadID): [Connector | null, boolean] {
|
|
849
|
-
const conn = this.connectors[id.toString()];
|
|
850
|
-
return [conn || null, !!conn];
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
/**
|
|
854
|
-
* Get thread offsets and peers
|
|
855
|
-
*/
|
|
856
|
-
async threadOffsets(
|
|
857
|
-
tid: ThreadID
|
|
858
|
-
): Promise<[Record<string, Head>, PeerId[]]> {
|
|
859
|
-
const info = await this.logstore.getThread(tid);
|
|
860
|
-
|
|
861
|
-
const offsets: Record<string, Head> = {};
|
|
862
|
-
const addrs: TMultiaddr[] = [];
|
|
863
|
-
|
|
864
|
-
// Process all logs in thread
|
|
865
|
-
for (const lg of info.logs) {
|
|
866
|
-
// Check if head is known
|
|
867
|
-
let has = false;
|
|
868
|
-
|
|
869
|
-
if (lg.head?.id) {
|
|
870
|
-
has = await this.isKnown(lg.head.id);
|
|
871
|
-
}
|
|
872
|
-
|
|
873
|
-
if (has && lg.head) {
|
|
874
|
-
offsets[lg.id.toString()] = lg.head;
|
|
875
|
-
// Collect addresses
|
|
876
|
-
} else {
|
|
877
|
-
offsets[lg.id.toString()] = await getHeadUndef();
|
|
878
|
-
}
|
|
879
|
-
if (lg.addrs && lg.addrs.length > 0) {
|
|
880
|
-
addrs.push(...lg.addrs);
|
|
881
|
-
}
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
// Get unique peer IDs
|
|
885
|
-
const peers = await this.uniquePeers(addrs);
|
|
886
|
-
|
|
887
|
-
return [offsets, peers];
|
|
888
|
-
}
|
|
889
|
-
|
|
890
|
-
async updateLogsFromPeer(tid: ThreadID, peerId: PeerId): Promise<void> {
|
|
891
|
-
try {
|
|
892
|
-
const [client, _] = await this.getClient(peerId);
|
|
893
|
-
if (!client) {
|
|
894
|
-
return;
|
|
895
|
-
}
|
|
896
|
-
const dbClient = new DBClient(client, this.dc, this, this.logstore);
|
|
897
|
-
await dbClient.scheduleUpdateLogs(tid);
|
|
898
|
-
} catch (err) {
|
|
899
|
-
throw new Error(
|
|
900
|
-
`Getting records for thread ${tid} failed: ${
|
|
901
|
-
err instanceof Error ? err.message : String(err)
|
|
902
|
-
}`
|
|
903
|
-
);
|
|
904
|
-
}
|
|
905
|
-
}
|
|
906
|
-
|
|
907
|
-
/**
|
|
908
|
-
* 从对等点获取新的日志和记录并添加到本地存储
|
|
909
|
-
*
|
|
910
|
-
* @param tid threaddb ID
|
|
911
|
-
* @param pid 对等点ID
|
|
912
|
-
* @returns 无返回值,但会抛出错误
|
|
913
|
-
*/
|
|
914
|
-
async updateRecordsFromPeer(
|
|
915
|
-
tid: ThreadID,
|
|
916
|
-
peerId: PeerId | null,
|
|
917
|
-
client?: DBClient
|
|
918
|
-
): Promise<void> {
|
|
919
|
-
try {
|
|
920
|
-
// 获取threaddb 偏移量
|
|
921
|
-
const [offsets, peers] = await this.threadOffsets(tid);
|
|
922
|
-
|
|
923
|
-
// 构建获取记录的请求
|
|
924
|
-
const { req, serviceKey } = await this.buildGetRecordsRequest(
|
|
925
|
-
tid,
|
|
926
|
-
offsets,
|
|
927
|
-
netPullingLimit
|
|
928
|
-
);
|
|
929
|
-
let recs: Record<string, PeerRecords> = {};
|
|
930
|
-
if (client) {
|
|
931
|
-
recs = await this.getRecordsWithDbClient(client, req, serviceKey);
|
|
932
|
-
} else {
|
|
933
|
-
if (!peerId) {
|
|
934
|
-
throw new Error("A peer-id is required to request records");
|
|
935
|
-
}
|
|
936
|
-
// 从对等点获取记录
|
|
937
|
-
recs = await this.getRecordsFromPeer(peerId, req, serviceKey);
|
|
938
|
-
}
|
|
939
|
-
// 处理接收到的记录
|
|
940
|
-
for (const [lidStr, rs] of Object.entries(recs)) {
|
|
941
|
-
try {
|
|
942
|
-
// 将字符串ID转换为PeerId对象
|
|
943
|
-
const lid = peerIdFromString(lidStr);
|
|
944
|
-
|
|
945
|
-
// 将记录添加到本地存储
|
|
946
|
-
await this.putRecords(tid, lid, rs.records, rs.counter);
|
|
947
|
-
} catch (err) {
|
|
948
|
-
throw new Error(
|
|
949
|
-
`Putting records from log ${lidStr} (thread ${tid}) failed: ${
|
|
950
|
-
err instanceof Error ? err.message : String(err)
|
|
951
|
-
}`
|
|
952
|
-
);
|
|
953
|
-
}
|
|
954
|
-
}
|
|
955
|
-
// 检查是否可能有更多记录需要获取
|
|
956
|
-
for (const [lidStr, rs] of Object.entries(recs)) {
|
|
957
|
-
try {
|
|
958
|
-
const lid = peerIdFromString(lidStr);
|
|
959
|
-
const head = await this.currentHead(tid, lid);
|
|
960
|
-
|
|
961
|
-
// 如果我们收到了最大数量的记录,并且可能还有更多
|
|
962
|
-
if (
|
|
963
|
-
head.counter <= rs.counter &&
|
|
964
|
-
rs.records.length === netPullingLimit
|
|
965
|
-
) {
|
|
966
|
-
// 递归调用继续获取更多记录
|
|
967
|
-
return this.updateRecordsFromPeer(tid, peerId, client);
|
|
968
|
-
}
|
|
969
|
-
} catch (err) {
|
|
970
|
-
// 忽略获取头部的错误,继续检查其他日志
|
|
971
|
-
console.warn(
|
|
972
|
-
`Error checking head for log ${lidStr}: ${
|
|
973
|
-
err instanceof Error ? err.message : String(err)
|
|
974
|
-
}`
|
|
975
|
-
);
|
|
976
|
-
}
|
|
977
|
-
}
|
|
978
|
-
} catch (err) {
|
|
979
|
-
throw new Error(
|
|
980
|
-
`Getting records for thread ${tid} failed: ${
|
|
981
|
-
err instanceof Error ? err.message : String(err)
|
|
982
|
-
}`
|
|
983
|
-
);
|
|
984
|
-
}
|
|
985
|
-
}
|
|
986
|
-
|
|
987
|
-
/**
|
|
988
|
-
* 从特定对等点获取记录
|
|
989
|
-
* 注意: 这是一个假设的实现,需要根据你的实际gRPC客户端实现来调整
|
|
990
|
-
*/
|
|
991
|
-
async getRecordsFromPeer(
|
|
992
|
-
peerId: PeerId,
|
|
993
|
-
req: any,
|
|
994
|
-
serviceKey: SymKey
|
|
995
|
-
): Promise<Record<string, PeerRecords>> {
|
|
996
|
-
try {
|
|
997
|
-
const [client, _] = await this.getClient(peerId);
|
|
998
|
-
if (!client) {
|
|
999
|
-
return {};
|
|
1000
|
-
}
|
|
1001
|
-
const dbClient = new DBClient(client, this.dc, this, this.logstore);
|
|
1002
|
-
const recs = await dbClient.getRecordsFromPeer(req, serviceKey);
|
|
1003
|
-
return recs;
|
|
1004
|
-
} catch (err) {
|
|
1005
|
-
console.error("getRecordsFromPeer error:", err);
|
|
1006
|
-
throw err;
|
|
1007
|
-
}
|
|
1008
|
-
}
|
|
1009
|
-
|
|
1010
|
-
async getRecordsWithDbClient(
|
|
1011
|
-
dbClient: DBClient,
|
|
1012
|
-
req: any,
|
|
1013
|
-
serviceKey: SymKey
|
|
1014
|
-
): Promise<Record<string, PeerRecords>> {
|
|
1015
|
-
try {
|
|
1016
|
-
const recs = await dbClient.getRecordsFromPeer(req, serviceKey);
|
|
1017
|
-
return recs;
|
|
1018
|
-
} catch (err) {
|
|
1019
|
-
console.error("getRecordsFromPeer error:", err);
|
|
1020
|
-
throw err;
|
|
1021
|
-
}
|
|
1022
|
-
}
|
|
1023
|
-
|
|
1024
|
-
/**
|
|
1025
|
-
* Add records to a thread
|
|
1026
|
-
*/
|
|
1027
|
-
async putRecords(
|
|
1028
|
-
tid: ThreadID,
|
|
1029
|
-
lid: PeerId,
|
|
1030
|
-
recs: IRecord[],
|
|
1031
|
-
counter: number
|
|
1032
|
-
): Promise<void> {
|
|
1033
|
-
const [chain, head] = await this.loadRecords(tid, lid, recs, counter);
|
|
1034
|
-
|
|
1035
|
-
if (chain.length === 0) {
|
|
1036
|
-
return;
|
|
1037
|
-
}
|
|
1038
|
-
const mutex = this.getMutexForThread(tid.toString());
|
|
1039
|
-
await mutex.acquire();
|
|
1040
|
-
try {
|
|
1041
|
-
// Check the head again, as another process could have changed the log
|
|
1042
|
-
const current = await this.currentHead(tid, lid);
|
|
1043
|
-
let headReached = true;
|
|
1044
|
-
let updatedHead = head;
|
|
1045
|
-
if (current?.id?.toString() != head?.id?.toString()) {
|
|
1046
|
-
// Fast-forward the chain up to the updated head
|
|
1047
|
-
headReached = false;
|
|
1048
|
-
updatedHead = current;
|
|
1049
|
-
|
|
1050
|
-
for (let i = 0; i < chain.length; i++) {
|
|
1051
|
-
if (chain[i]!.value().cid().equals(current.id)) {
|
|
1052
|
-
chain.splice(0, i + 1);
|
|
1053
|
-
headReached = true;
|
|
1054
|
-
break;
|
|
1055
|
-
}
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
if (!headReached) {
|
|
1059
|
-
// Entire chain already processed
|
|
1060
|
-
return;
|
|
1061
|
-
}
|
|
1062
|
-
}
|
|
1063
|
-
|
|
1064
|
-
const [connector, appConnected] = this.getConnector(tid);
|
|
1065
|
-
let identity: Ed25519PubKey;
|
|
1066
|
-
let readKey: SymmetricKey | null = null;
|
|
1067
|
-
let validate = false;
|
|
1068
|
-
let updatedCounter = updatedHead.counter;
|
|
1069
|
-
|
|
1070
|
-
if (appConnected) {
|
|
1071
|
-
const symKey = await this.logstore.keyBook.readKey(tid);
|
|
1072
|
-
if (symKey) {
|
|
1073
|
-
readKey = SymmetricKey.fromSymKey(symKey);
|
|
1074
|
-
}
|
|
1075
|
-
if (readKey) {
|
|
1076
|
-
validate = true;
|
|
1077
|
-
}
|
|
1078
|
-
}
|
|
1079
|
-
for (const record of chain) {
|
|
1080
|
-
if (validate) {
|
|
1081
|
-
// Validate the record
|
|
1082
|
-
const block = await record.value().getBlock(this.bstore);
|
|
1083
|
-
|
|
1084
|
-
let event: Event;
|
|
1085
|
-
if (block instanceof Event) {
|
|
1086
|
-
event = block;
|
|
1087
|
-
} else {
|
|
1088
|
-
event = (await EventFromNode(block as Node)) as Event;
|
|
1089
|
-
}
|
|
1090
|
-
|
|
1091
|
-
const dbody = await event.getBody(
|
|
1092
|
-
this.bstore,
|
|
1093
|
-
readKey ? readKey : undefined
|
|
1094
|
-
);
|
|
1095
|
-
|
|
1096
|
-
identity = await KeyConverter.publicFromBytes<Ed25519PubKey>(
|
|
1097
|
-
record.value().pubKey()
|
|
1098
|
-
);
|
|
1099
|
-
|
|
1100
|
-
try {
|
|
1101
|
-
await connector!.validateNetRecordBody(dbody, identity);
|
|
1102
|
-
} catch (err) {
|
|
1103
|
-
// If validation fails, clean up blocks
|
|
1104
|
-
const header = await event.getHeader(this.bstore);
|
|
1105
|
-
const body = await event.getBody(this.bstore);
|
|
1106
|
-
this.bstore.deleteMany([event.cid(), header.cid(), body.cid()]);
|
|
1107
|
-
throw err;
|
|
1108
|
-
}
|
|
1109
|
-
}
|
|
1110
|
-
|
|
1111
|
-
// Update head counter
|
|
1112
|
-
updatedCounter++;
|
|
1113
|
-
|
|
1114
|
-
// Set new head for the log
|
|
1115
|
-
await this.logstore.headBook.setHead(tid, lid, {
|
|
1116
|
-
id: record.value().cid(),
|
|
1117
|
-
counter: updatedCounter,
|
|
1118
|
-
});
|
|
1119
|
-
|
|
1120
|
-
// Set checkpoint for log
|
|
1121
|
-
await this.setThreadLogPoint(
|
|
1122
|
-
tid,
|
|
1123
|
-
lid,
|
|
1124
|
-
updatedCounter,
|
|
1125
|
-
record.value().cid()
|
|
1126
|
-
);
|
|
1127
|
-
|
|
1128
|
-
// Handle record in app connector
|
|
1129
|
-
if (appConnected) {
|
|
1130
|
-
try {
|
|
1131
|
-
await connector!.handleNetRecord(record);
|
|
1132
|
-
} catch (err) {
|
|
1133
|
-
// Continue processing subsequent records
|
|
1134
|
-
continue;
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
// Add record to blockstore
|
|
1139
|
-
await this.bstore.put(record.value().cid(), record.value().data());
|
|
1140
|
-
}
|
|
1141
|
-
} finally {
|
|
1142
|
-
mutex.release();
|
|
1143
|
-
}
|
|
1144
|
-
}
|
|
1145
|
-
|
|
1146
|
-
/**
|
|
1147
|
-
* Load records from a thread
|
|
1148
|
-
*/
|
|
1149
|
-
async loadRecords(
|
|
1150
|
-
tid: ThreadID,
|
|
1151
|
-
lid: PeerId,
|
|
1152
|
-
recs: IRecord[],
|
|
1153
|
-
counter: number
|
|
1154
|
-
): Promise<[IThreadRecord[], Head]> {
|
|
1155
|
-
if (recs.length === 0) {
|
|
1156
|
-
throw new Error("Cannot load empty record chain");
|
|
1157
|
-
}
|
|
1158
|
-
|
|
1159
|
-
const head = await this.currentHead(tid, lid);
|
|
1160
|
-
|
|
1161
|
-
// Check if the last record was already loaded and processed
|
|
1162
|
-
const last = recs[recs.length - 1]!;
|
|
1163
|
-
|
|
1164
|
-
// If we don't have the counter, check if record exists
|
|
1165
|
-
if (counter === undefined) {
|
|
1166
|
-
const exist = await this.isKnown(last.cid());
|
|
1167
|
-
if (exist || !(last.cid().toString() == "")) {
|
|
1168
|
-
return [[], head];
|
|
1169
|
-
}
|
|
1170
|
-
} else if (counter <= head.counter) {
|
|
1171
|
-
return [[], head];
|
|
1172
|
-
}
|
|
1173
|
-
let chain: IRecord[] = [];
|
|
1174
|
-
let complete = false;
|
|
1175
|
-
// Check which records we already have
|
|
1176
|
-
for (let i = recs.length - 1; i >= 0; i--) {
|
|
1177
|
-
const next = recs[i]!;
|
|
1178
|
-
if (next.cid().toString() == "" || next.cid().equals(head.id)) {
|
|
1179
|
-
complete = true;
|
|
1180
|
-
break;
|
|
1181
|
-
}
|
|
1182
|
-
chain.push(next);
|
|
1183
|
-
}
|
|
1184
|
-
|
|
1185
|
-
// Bridge the gap between the last provided record and current head
|
|
1186
|
-
if (!complete && chain.length > 0) {
|
|
1187
|
-
|
|
1188
|
-
let c = chain[chain.length - 1]!.prevID();
|
|
1189
|
-
while (c && !(last.cid().toString() == "")) {
|
|
1190
|
-
if (c.equals(head.id)) {
|
|
1191
|
-
break;
|
|
1192
|
-
}
|
|
1193
|
-
const r = await this.getRecord(tid, c);
|
|
1194
|
-
chain.push(r);
|
|
1195
|
-
c = r.prevID();
|
|
1196
|
-
}
|
|
1197
|
-
}
|
|
1198
|
-
|
|
1199
|
-
if (chain.length === 0) {
|
|
1200
|
-
return [[], head];
|
|
1201
|
-
}
|
|
1202
|
-
|
|
1203
|
-
// Prepare thread records in the right order
|
|
1204
|
-
const tRecords: ThreadRecord[] = [];
|
|
1205
|
-
|
|
1206
|
-
for (let i = chain.length - 1; i >= 0; i--) {
|
|
1207
|
-
const r = chain[i]!;
|
|
1208
|
-
|
|
1209
|
-
// Get and cache blocks
|
|
1210
|
-
const block = await r.getBlock(this.bstore);
|
|
1211
|
-
let event: IThreadEvent;
|
|
1212
|
-
if (block instanceof Event) {
|
|
1213
|
-
event = block;
|
|
1214
|
-
} else {
|
|
1215
|
-
event = await EventFromNode(block as Node);
|
|
1216
|
-
}
|
|
1217
|
-
|
|
1218
|
-
const header = await event.getHeader(this.bstore);
|
|
1219
|
-
const body = await event.getBody(this.bstore);
|
|
1220
|
-
|
|
1221
|
-
// Store internal blocks
|
|
1222
|
-
await this.addMany([event, header, body]);
|
|
1223
|
-
tRecords.push(newRecord(r, tid, lid) as ThreadRecord);
|
|
1224
|
-
}
|
|
1225
|
-
return [tRecords, head];
|
|
1226
|
-
}
|
|
1227
|
-
|
|
1228
|
-
/**
|
|
1229
|
-
* Check if a record exists
|
|
1230
|
-
*/
|
|
1231
|
-
async isKnown(rec: CID): Promise<boolean> {
|
|
1232
|
-
return await this.bstore.has(rec);
|
|
1233
|
-
}
|
|
1234
|
-
|
|
1235
|
-
/**
|
|
1236
|
-
* 获取记录
|
|
1237
|
-
* 从给定的threaddb 和CID获取记录
|
|
1238
|
-
*
|
|
1239
|
-
* @param ctx 上下文
|
|
1240
|
-
* @param id threaddb ID
|
|
1241
|
-
* @param rid 记录CID
|
|
1242
|
-
* @returns 检索到的记录
|
|
1243
|
-
* @throws 如果无法获取服务密钥或记录
|
|
1244
|
-
*/
|
|
1245
|
-
async getRecord(id: ThreadID, rid: CID): Promise<IRecord> {
|
|
1246
|
-
// 从存储中获取服务密钥
|
|
1247
|
-
const serviceKey = await this.logstore.keyBook.serviceKey(id);
|
|
1248
|
-
|
|
1249
|
-
// 检查服务密钥是否存在
|
|
1250
|
-
if (!serviceKey) {
|
|
1251
|
-
throw new Error("A service-key is required to get records");
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
const sk = SymmetricKey.fromSymKey(serviceKey);
|
|
1255
|
-
// 使用CBOR获取记录
|
|
1256
|
-
return await GetRecord(this.dagService, rid, sk);
|
|
1257
|
-
}
|
|
1258
|
-
/**
|
|
1259
|
-
* Set a thread log checkpoint
|
|
1260
|
-
*/
|
|
1261
|
-
async setThreadLogPoint(
|
|
1262
|
-
tid: ThreadID,
|
|
1263
|
-
lid: PeerId,
|
|
1264
|
-
counter: number,
|
|
1265
|
-
rcid: CID
|
|
1266
|
-
): Promise<void> {
|
|
1267
|
-
// Only store checkpoints at multiples of 10000
|
|
1268
|
-
if (counter % 10000 !== 0) {
|
|
1269
|
-
return;
|
|
1270
|
-
}
|
|
1271
|
-
// Create key for checkpoint
|
|
1272
|
-
const key = `${lid.toString()}/${counter}`;
|
|
1273
|
-
|
|
1274
|
-
// Store checkpoint
|
|
1275
|
-
await this.logstore.metadata.putBytes(tid, key, rcid.bytes);
|
|
1276
|
-
}
|
|
1277
|
-
|
|
1278
|
-
/**
|
|
1279
|
-
* Get the current head of a log
|
|
1280
|
-
*/
|
|
1281
|
-
async currentHead(tid: ThreadID, lid: PeerId): Promise<Head> {
|
|
1282
|
-
const heads = await this.logstore.headBook.heads(tid, lid);
|
|
1283
|
-
|
|
1284
|
-
if (heads.length > 0) {
|
|
1285
|
-
return heads[0]!;
|
|
1286
|
-
} else {
|
|
1287
|
-
return await getHeadUndef();
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
|
-
/**
|
|
1291
|
-
* Extract unique peer IDs from addresses
|
|
1292
|
-
*/
|
|
1293
|
-
async uniquePeers(addrs: TMultiaddr[]): Promise<PeerId[]> {
|
|
1294
|
-
const peerMap = new Map<string, PeerId>();
|
|
1295
|
-
|
|
1296
|
-
for (const addr of addrs) {
|
|
1297
|
-
try {
|
|
1298
|
-
const [pid, callable] = await this.callablePeer(addr);
|
|
1299
|
-
|
|
1300
|
-
if (!callable) {
|
|
1301
|
-
continue;
|
|
1302
|
-
}
|
|
1303
|
-
|
|
1304
|
-
peerMap.set(pid.toString(), pid);
|
|
1305
|
-
} catch (err) {
|
|
1306
|
-
// Skip addresses that can't be parsed
|
|
1307
|
-
}
|
|
1308
|
-
}
|
|
1309
|
-
|
|
1310
|
-
return Array.from(peerMap.values());
|
|
1311
|
-
}
|
|
1312
|
-
|
|
1313
|
-
/**
|
|
1314
|
-
* Extract peer ID from multiaddress and check if callable
|
|
1315
|
-
*/
|
|
1316
|
-
async callablePeer(addr: TMultiaddr): Promise<[PeerId, boolean]> {
|
|
1317
|
-
const p = addr.getPeerId();
|
|
1318
|
-
if (!p) {
|
|
1319
|
-
throw new Error("Address does not contain peer ID");
|
|
1320
|
-
}
|
|
1321
|
-
|
|
1322
|
-
const pid = peerIdFromString(p);
|
|
1323
|
-
|
|
1324
|
-
// Don't call ourselves
|
|
1325
|
-
if (pid.toString() === this.hostID) {
|
|
1326
|
-
return [pid, false];
|
|
1327
|
-
}
|
|
1328
|
-
return [pid, true];
|
|
1329
|
-
}
|
|
1330
|
-
|
|
1331
|
-
/**
|
|
1332
|
-
* 从指定对等点获取记录
|
|
1333
|
-
*/
|
|
1334
|
-
async getRecords(
|
|
1335
|
-
peers: PeerId[],
|
|
1336
|
-
tid: ThreadID,
|
|
1337
|
-
offsets: Record<string, { id?: CID, counter: number }>,
|
|
1338
|
-
limit: number,
|
|
1339
|
-
multiPeersFlag: boolean = false
|
|
1340
|
-
): Promise<Record<string, PeerRecords>> {
|
|
1341
|
-
try {
|
|
1342
|
-
// 构建请求
|
|
1343
|
-
const { req, serviceKey } = await this.buildGetRecordsRequest(tid, offsets, limit);
|
|
1344
|
-
|
|
1345
|
-
// 创建记录收集器
|
|
1346
|
-
const recordCollector = new RecordCollector();
|
|
1347
|
-
// 设置超时
|
|
1348
|
-
let timeout : NodeJS.Timeout | null = null;
|
|
1349
|
-
//遍历节点,按顺序处理
|
|
1350
|
-
for (const peerId of peers) {
|
|
1351
|
-
try {
|
|
1352
|
-
timeout = setTimeout(() => {
|
|
1353
|
-
throw new Error(`Timeout getting records from peer ${peerId}`);
|
|
1354
|
-
}, 900000);
|
|
1355
|
-
//连接到指定peerId,返回一个Client
|
|
1356
|
-
const [client,err] = await this.getClient(peerId);
|
|
1357
|
-
if (!client) {
|
|
1358
|
-
throw new Error(`Error getting records from peer ${peerId},no client,errinfo: ${err}`);
|
|
1359
|
-
}
|
|
1360
|
-
console.log(`时间:${new Date().toLocaleString()} ,开始从 ${peerId.toString()} 获取记录...`);
|
|
1361
|
-
const dbClient = new DBClient(client,this.dc,this,this.logstore);
|
|
1362
|
-
const records = await dbClient.getRecordsFromPeer( req, serviceKey);
|
|
1363
|
-
console.log(`时间:${new Date().toLocaleString()} ,从 ${peerId.toString()} 获取记录完成,记录数为:`,Object.keys(records).length);
|
|
1364
|
-
console.log(`开始处理从 ${peerId.toString()} 获取的记录,记录数为:`,Object.keys(records).length);
|
|
1365
|
-
for (const [logId, rs] of Object.entries(records)) {
|
|
1366
|
-
await recordCollector.batchUpdate(logId, rs);
|
|
1367
|
-
}
|
|
1368
|
-
console.log(`处理从 ${peerId.toString()} 获取的记录完成,记录数为:`,Object.keys(records).length);
|
|
1369
|
-
|
|
1370
|
-
if(!multiPeersFlag){
|
|
1371
|
-
break;
|
|
1372
|
-
}
|
|
1373
|
-
} catch (err) {
|
|
1374
|
-
continue;
|
|
1375
|
-
}finally{
|
|
1376
|
-
// 清除超时
|
|
1377
|
-
if (timeout) {
|
|
1378
|
-
clearTimeout(timeout);
|
|
1379
|
-
}
|
|
1380
|
-
}
|
|
1381
|
-
}
|
|
1382
|
-
|
|
1383
|
-
return recordCollector.list();
|
|
1384
|
-
} catch (err) {
|
|
1385
|
-
console.error("getRecords error:", err);
|
|
1386
|
-
throw err;
|
|
1387
|
-
}
|
|
1388
|
-
}
|
|
1389
|
-
|
|
1390
|
-
/**
|
|
1391
|
-
* 构建获取记录的请求
|
|
1392
|
-
*/
|
|
1393
|
-
async buildGetRecordsRequest(
|
|
1394
|
-
tid: ThreadID,
|
|
1395
|
-
offsets: Record<string, Head>,
|
|
1396
|
-
limit: number
|
|
1397
|
-
): Promise<{ req: net_pb.pb.IGetRecordsRequest; serviceKey: SymKey }> {
|
|
1398
|
-
try {
|
|
1399
|
-
// 从存储中获取服务密钥
|
|
1400
|
-
const serviceKey = await this.logstore.keyBook.serviceKey(tid);
|
|
1401
|
-
|
|
1402
|
-
if (!serviceKey) {
|
|
1403
|
-
throw new Error("A service-key is required to request records");
|
|
1404
|
-
}
|
|
1405
|
-
|
|
1406
|
-
// 创建日志条目
|
|
1407
|
-
const logs = Object.entries(offsets).map(([logId, offset]) => {
|
|
1408
|
-
const pbLog: net_pb.pb.GetRecordsRequest.Body.ILogEntry = {
|
|
1409
|
-
logID: PeerIDConverter.toBytes(logId),
|
|
1410
|
-
limit: limit,
|
|
1411
|
-
counter: offset.counter,
|
|
1412
|
-
offset: offset.id
|
|
1413
|
-
? CidConverter.toBytes(offset.id)
|
|
1414
|
-
: new Uint8Array(),
|
|
1415
|
-
};
|
|
1416
|
-
return pbLog;
|
|
1417
|
-
});
|
|
1418
|
-
|
|
1419
|
-
// 构建请求体
|
|
1420
|
-
const body: net_pb.pb.GetRecordsRequest.IBody = {
|
|
1421
|
-
threadID: ThreadIDConverter.toBytes(tid.toString()),
|
|
1422
|
-
serviceKey: serviceKey.raw,
|
|
1423
|
-
logs: logs,
|
|
1424
|
-
};
|
|
1425
|
-
|
|
1426
|
-
// 创建请求
|
|
1427
|
-
const req = new net_pb.pb.GetRecordsRequest();
|
|
1428
|
-
req.body = body;
|
|
1429
|
-
return { req, serviceKey };
|
|
1430
|
-
} catch (err) {
|
|
1431
|
-
console.error("buildGetRecordsRequest error:", err);
|
|
1432
|
-
throw err;
|
|
1433
|
-
}
|
|
1434
|
-
}
|
|
1435
|
-
|
|
1436
|
-
/**
|
|
1437
|
-
* 连接应用到指定threaddb
|
|
1438
|
-
* @param app 应用程序对象
|
|
1439
|
-
* @param id threaddb ID
|
|
1440
|
-
* @returns 返回一个应用连接器
|
|
1441
|
-
* @throws 如果验证失败或获取threaddb 信息出错则抛出异常
|
|
1442
|
-
*/
|
|
1443
|
-
async connectApp(app: App, id: ThreadID): Promise<Connector> {
|
|
1444
|
-
// 验证threaddb ID
|
|
1445
|
-
if (!id.isDefined()) {
|
|
1446
|
-
throw new Error("Invalid thread ID");
|
|
1447
|
-
}
|
|
1448
|
-
// 获取threaddb信息
|
|
1449
|
-
let info;
|
|
1450
|
-
try {
|
|
1451
|
-
info = await this.getThreadWithAddrs(id);
|
|
1452
|
-
} catch (err) {
|
|
1453
|
-
throw new Error(
|
|
1454
|
-
`Error getting thread ${id.toString()}: ${
|
|
1455
|
-
err instanceof Error ? err.message : String(err)
|
|
1456
|
-
}`
|
|
1457
|
-
);
|
|
1458
|
-
}
|
|
1459
|
-
|
|
1460
|
-
// 创建应用连接器
|
|
1461
|
-
let con;
|
|
1462
|
-
try {
|
|
1463
|
-
con = new Connector(this, app, info);
|
|
1464
|
-
} catch (err) {
|
|
1465
|
-
throw new Error(
|
|
1466
|
-
`Error making connector ${id.toString()}: ${
|
|
1467
|
-
err instanceof Error ? err.message : String(err)
|
|
1468
|
-
}`
|
|
1469
|
-
);
|
|
1470
|
-
}
|
|
1471
|
-
|
|
1472
|
-
// 添加连接器到网络
|
|
1473
|
-
this.addConnector(id, con);
|
|
1474
|
-
|
|
1475
|
-
return con;
|
|
1476
|
-
}
|
|
1477
|
-
|
|
1478
|
-
/**
|
|
1479
|
-
* 获取线程的所有日志
|
|
1480
|
-
* 返回线程的日志列表和线程信息
|
|
1481
|
-
*
|
|
1482
|
-
* @param tid 线程ID
|
|
1483
|
-
* @returns 日志数组和线程信息
|
|
1484
|
-
*/
|
|
1485
|
-
async getPbLogs(tid: ThreadID): Promise<[net_pb.pb.ILog[], IThreadInfo]> {
|
|
1486
|
-
try {
|
|
1487
|
-
// 从存储中获取线程信息
|
|
1488
|
-
const info = await this.logstore.getThread(tid);
|
|
1489
|
-
|
|
1490
|
-
// 创建日志数组
|
|
1491
|
-
const logs: net_pb.pb.ILog[] = [];
|
|
1492
|
-
|
|
1493
|
-
// 将每个日志信息转换为protobuf格式
|
|
1494
|
-
for (const logInfo of info.logs) {
|
|
1495
|
-
logs.push(await logToProto(logInfo));
|
|
1496
|
-
}
|
|
1497
|
-
return [logs, info];
|
|
1498
|
-
} catch (err) {
|
|
1499
|
-
throw new Error(
|
|
1500
|
-
`Failed to get logs for thread ${tid}: ${
|
|
1501
|
-
err instanceof Error ? err.message : String(err)
|
|
1502
|
-
}`
|
|
1503
|
-
);
|
|
1504
|
-
}
|
|
1505
|
-
}
|
|
1506
|
-
|
|
1507
|
-
/**
|
|
1508
|
-
* 预加载日志到线程
|
|
1509
|
-
* 浏览器兼容版本
|
|
1510
|
-
*
|
|
1511
|
-
* @param tid 线程ID
|
|
1512
|
-
* @param logs 协议缓冲区日志数组
|
|
1513
|
-
*/
|
|
1514
|
-
async preLoadLogs(tid: ThreadID, logs: net_pb.pb.Log[]): Promise<void> {
|
|
1515
|
-
try {
|
|
1516
|
-
// 创建一个与日志数组相同长度的ThreadLogInfo数组
|
|
1517
|
-
const lgs: IThreadLogInfo[] = [];
|
|
1518
|
-
|
|
1519
|
-
// 遍历并转换每个日志
|
|
1520
|
-
for (let i = 0; i < logs.length; i++) {
|
|
1521
|
-
lgs.push(await logFromProto(logs[i]!));
|
|
1522
|
-
}
|
|
1523
|
-
// 调用创建外部日志的方法
|
|
1524
|
-
await this.createExternalLogsIfNotExistForPreload(tid, lgs);
|
|
1525
|
-
} catch (err) {
|
|
1526
|
-
throw new Error(
|
|
1527
|
-
`Failed to preload logs: ${
|
|
1528
|
-
err instanceof Error ? err.message : String(err)
|
|
1529
|
-
}`
|
|
1530
|
-
);
|
|
1531
|
-
}
|
|
1532
|
-
}
|
|
1533
|
-
|
|
1534
|
-
/**
|
|
1535
|
-
* 创建外部日志(如果不存在)
|
|
1536
|
-
* 创建的日志将使用未定义CID作为当前头部
|
|
1537
|
-
* 此方法是线程安全的
|
|
1538
|
-
*
|
|
1539
|
-
* @param tid 线程ID
|
|
1540
|
-
* @param logs 日志信息数组
|
|
1541
|
-
*/
|
|
1542
|
-
async createExternalLogsIfNotExist(
|
|
1543
|
-
tid: ThreadID,
|
|
1544
|
-
logs: IThreadLogInfo[]
|
|
1545
|
-
): Promise<void> {
|
|
1546
|
-
const mutex = this.getMutexForThread(tid.toString());
|
|
1547
|
-
await mutex.acquire();
|
|
1548
|
-
try {
|
|
1549
|
-
for (const log of logs) {
|
|
1550
|
-
try {
|
|
1551
|
-
const heads = await this.logstore.headBook.heads(tid, log.id);
|
|
1552
|
-
|
|
1553
|
-
if (heads.length === 0) {
|
|
1554
|
-
log.head = await getHeadUndef();
|
|
1555
|
-
await this.logstore.addLog(tid, log);
|
|
1556
|
-
} else {
|
|
1557
|
-
await this.logstore.addrBook.addAddrs(
|
|
1558
|
-
tid,
|
|
1559
|
-
log.id,
|
|
1560
|
-
log.addrs,
|
|
1561
|
-
PermanentAddrTTL
|
|
1562
|
-
);
|
|
1563
|
-
}
|
|
1564
|
-
} catch (err) {
|
|
1565
|
-
continue;
|
|
1566
|
-
}
|
|
1567
|
-
}
|
|
1568
|
-
} finally {
|
|
1569
|
-
mutex.release();
|
|
1570
|
-
}
|
|
1571
|
-
}
|
|
1572
|
-
|
|
1573
|
-
/**
|
|
1574
|
-
* 预加载时创建外部日志(如果不存在)
|
|
1575
|
-
* 浏览器兼容版本
|
|
1576
|
-
*
|
|
1577
|
-
* @param tid 线程ID
|
|
1578
|
-
* @param logs 日志信息数组
|
|
1579
|
-
*/
|
|
1580
|
-
async createExternalLogsIfNotExistForPreload(
|
|
1581
|
-
tid: ThreadID,
|
|
1582
|
-
logs: IThreadLogInfo[]
|
|
1583
|
-
): Promise<void> {
|
|
1584
|
-
const mutex = this.getMutexForThread(tid.toString());
|
|
1585
|
-
await mutex.acquire();
|
|
1586
|
-
try {
|
|
1587
|
-
for (const log of logs) {
|
|
1588
|
-
try {
|
|
1589
|
-
const heads = await this.logstore.headBook.heads(tid, log.id);
|
|
1590
|
-
if (heads.length === 0) {
|
|
1591
|
-
await this.logstore.addLog(tid, log);
|
|
1592
|
-
} else {
|
|
1593
|
-
await this.logstore.addrBook.addAddrs(
|
|
1594
|
-
tid,
|
|
1595
|
-
log.id,
|
|
1596
|
-
log.addrs,
|
|
1597
|
-
PermanentAddrTTL
|
|
1598
|
-
);
|
|
1599
|
-
}
|
|
1600
|
-
} catch (err) {
|
|
1601
|
-
continue;
|
|
1602
|
-
}
|
|
1603
|
-
}
|
|
1604
|
-
} finally {
|
|
1605
|
-
mutex.release();
|
|
1606
|
-
}
|
|
1607
|
-
}
|
|
1608
|
-
|
|
1609
|
-
/**
|
|
1610
|
-
* 创建新的记录
|
|
1611
|
-
*
|
|
1612
|
-
* @param id threadID
|
|
1613
|
-
* @param body 节点内容
|
|
1614
|
-
* @param options 选项
|
|
1615
|
-
* @returns 创建的threaddb记录
|
|
1616
|
-
*/
|
|
1617
|
-
async createRecord(
|
|
1618
|
-
id: ThreadID,
|
|
1619
|
-
body: IPLDNode,
|
|
1620
|
-
options: { token?: ThreadToken; apiToken?: Token } = {}
|
|
1621
|
-
): Promise<IThreadRecord> {
|
|
1622
|
-
try {
|
|
1623
|
-
if (this.context.publicKey === undefined) {
|
|
1624
|
-
throw new Error("No identity provided for creating record");
|
|
1625
|
-
}
|
|
1626
|
-
// 验证身份,用节点的pubkey
|
|
1627
|
-
const identity = this.context.publicKey;
|
|
1628
|
-
// 获取并验证连接器
|
|
1629
|
-
const [con, ok] = this.getConnectorProtected(id, options.apiToken);
|
|
1630
|
-
|
|
1631
|
-
if (!ok) {
|
|
1632
|
-
throw new Error("Cannot create record: thread in use");
|
|
1633
|
-
} else if (con) {
|
|
1634
|
-
await con.validateNetRecordBody(body, identity as Ed25519PubKey);
|
|
1635
|
-
}
|
|
1636
|
-
|
|
1637
|
-
// 获取或创建日志
|
|
1638
|
-
const lg = await this.getOrCreateLog(id, identity);
|
|
1639
|
-
|
|
1640
|
-
// 创建新记录
|
|
1641
|
-
const r = await this.newRecord(id, lg, body, identity);
|
|
1642
|
-
|
|
1643
|
-
// 创建threaddb 记录
|
|
1644
|
-
const tr = newRecord(r, id, lg.id);
|
|
1645
|
-
if (!lg.head) {
|
|
1646
|
-
lg.head = {
|
|
1647
|
-
counter: 0,
|
|
1648
|
-
};
|
|
1649
|
-
}
|
|
1650
|
-
|
|
1651
|
-
// 更新头部信息
|
|
1652
|
-
const head: Head = {
|
|
1653
|
-
id: tr.value().cid(),
|
|
1654
|
-
counter: lg.head.counter + 1,
|
|
1655
|
-
};
|
|
1656
|
-
|
|
1657
|
-
await this.logstore.headBook.setHead(id, lg.id, head);
|
|
1658
|
-
|
|
1659
|
-
// 设置记录点(每10000个记录设置一个检查点)
|
|
1660
|
-
await this.setThreadLogPoint(
|
|
1661
|
-
id,
|
|
1662
|
-
lg.id,
|
|
1663
|
-
lg.head?.counter + 1,
|
|
1664
|
-
tr.value().cid()
|
|
1665
|
-
);
|
|
1666
|
-
|
|
1667
|
-
console.debug(
|
|
1668
|
-
`Created record ${tr.value().cid()} (thread=${id}, log=${lg.id})`
|
|
1669
|
-
);
|
|
1670
|
-
|
|
1671
|
-
// 推送记录到节点
|
|
1672
|
-
if (this.server) {
|
|
1673
|
-
this.pushRecord(id, lg.id, tr.value(), lg.head.counter + 1);
|
|
1674
|
-
}
|
|
1675
|
-
|
|
1676
|
-
return tr;
|
|
1677
|
-
} catch (error) {
|
|
1678
|
-
throw error;
|
|
1679
|
-
}
|
|
1680
|
-
}
|
|
1681
|
-
|
|
1682
|
-
/**
|
|
1683
|
-
* 获取或创建当前身份的日志
|
|
1684
|
-
*/
|
|
1685
|
-
async getOrCreateLog(
|
|
1686
|
-
id: ThreadID,
|
|
1687
|
-
identity: PublicKey
|
|
1688
|
-
): Promise<IThreadLogInfo> {
|
|
1689
|
-
// 默认使用当前主机身份,如果未提供
|
|
1690
|
-
if (!identity) {
|
|
1691
|
-
throw new Error("No identity provided");
|
|
1692
|
-
}
|
|
1693
|
-
|
|
1694
|
-
// 尝试获取此身份的现有日志ID
|
|
1695
|
-
const lidb = await this.logstore.metadata.getBytes(id, identity.toString());
|
|
1696
|
-
|
|
1697
|
-
// 检查旧式"自有"日志
|
|
1698
|
-
if (!lidb && identity.equals(this.context.publicKey)) {
|
|
1699
|
-
const thrd = await this.logstore.getThread(id);
|
|
1700
|
-
const ownLog = thrd.getFirstPrivKeyLog();
|
|
1701
|
-
if (ownLog) {
|
|
1702
|
-
return ownLog;
|
|
1703
|
-
}
|
|
1704
|
-
} else if (lidb) {
|
|
1705
|
-
// 获取现有日志
|
|
1706
|
-
const lidbstr = new TextDecoder().decode(lidb);
|
|
1707
|
-
const lid = peerIdFromString(lidbstr);
|
|
1708
|
-
return this.logstore.getLog(id, lid);
|
|
1709
|
-
}
|
|
1710
|
-
|
|
1711
|
-
// 创建新日志,如果不存在
|
|
1712
|
-
return this.createLog(id, undefined, identity);
|
|
1713
|
-
}
|
|
1714
|
-
|
|
1715
|
-
/**
|
|
1716
|
-
* 创建新记录
|
|
1717
|
-
*/
|
|
1718
|
-
async newRecord(
|
|
1719
|
-
id: ThreadID,
|
|
1720
|
-
lg: IThreadLogInfo,
|
|
1721
|
-
body: IPLDNode,
|
|
1722
|
-
identity: PublicKey
|
|
1723
|
-
): Promise<IRecord> {
|
|
1724
|
-
// 获取threaddb 服务密钥
|
|
1725
|
-
const serviceKey = await this.logstore.keyBook.serviceKey(id);
|
|
1726
|
-
if (!serviceKey) {
|
|
1727
|
-
throw new Error("No service key for thread");
|
|
1728
|
-
}
|
|
1729
|
-
if (!this.context.publicKey) {
|
|
1730
|
-
throw new Error("No identity provided for creating record");
|
|
1731
|
-
}
|
|
1732
|
-
const heads = await this.logstore.headBook.heads(id, lg.id);
|
|
1733
|
-
// 创建事件
|
|
1734
|
-
const sk = SymmetricKey.fromSymKey(serviceKey);
|
|
1735
|
-
const readKey = await this.logstore.keyBook.readKey(id);
|
|
1736
|
-
const rk = readKey ? SymmetricKey.fromSymKey(readKey) : null;
|
|
1737
|
-
if (!rk) {
|
|
1738
|
-
throw new Error("No read key for thread");
|
|
1739
|
-
}
|
|
1740
|
-
const event = await CreateEvent(this.bstore, body as Node, rk);
|
|
1741
|
-
// 将事件保存到存储
|
|
1742
|
-
await this.bstore.put(event.cid(), event.data());
|
|
1743
|
-
let prev: CID | undefined = undefined;
|
|
1744
|
-
if (heads && heads.length > 0) {
|
|
1745
|
-
prev = heads[0]!.id;
|
|
1746
|
-
}
|
|
1747
|
-
const rec = await CreateRecord(this.bstore, {
|
|
1748
|
-
block: event,
|
|
1749
|
-
prev: prev,
|
|
1750
|
-
key: lg.privKey as Ed25519PrivKey,
|
|
1751
|
-
pubKey: this.context.publicKey,
|
|
1752
|
-
serviceKey: sk,
|
|
1753
|
-
} as CreateRecordConfig);
|
|
1754
|
-
|
|
1755
|
-
return rec;
|
|
1756
|
-
}
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
// 启动队列处理任务(只需调用一次)
|
|
1761
|
-
private startPushWorker() {
|
|
1762
|
-
if (this.pushWorkerStarted) return;
|
|
1763
|
-
this.pushWorkerStarted = true;
|
|
1764
|
-
|
|
1765
|
-
const sleep = (ms: number) =>
|
|
1766
|
-
new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
1767
|
-
let haveRecordFlag = false;
|
|
1768
|
-
const loop = async () => {
|
|
1769
|
-
while (true) {
|
|
1770
|
-
try {
|
|
1771
|
-
const item = this.pushQueue.shift();
|
|
1772
|
-
if (!item) {
|
|
1773
|
-
await sleep(1000);
|
|
1774
|
-
continue;
|
|
1775
|
-
}
|
|
1776
|
-
try {
|
|
1777
|
-
await this._doPushRecord(item.tid, item.lid, item.rec, item.counter);
|
|
1778
|
-
if (!haveRecordFlag) {
|
|
1779
|
-
await this.logstore.metadata.putString(
|
|
1780
|
-
item.tid,
|
|
1781
|
-
"local_log_no_record_flag",
|
|
1782
|
-
""
|
|
1783
|
-
);
|
|
1784
|
-
haveRecordFlag = true;
|
|
1785
|
-
}
|
|
1786
|
-
|
|
1787
|
-
} catch (err) {
|
|
1788
|
-
console.error("pushRecord failed:", err);
|
|
1789
|
-
await sleep(200); // 简单退避,避免持续报错阻塞后续任务
|
|
1790
|
-
}
|
|
1791
|
-
} catch (err) {
|
|
1792
|
-
console.error("push worker crashed:", err);
|
|
1793
|
-
await sleep(500);
|
|
1794
|
-
}
|
|
1795
|
-
}
|
|
1796
|
-
};
|
|
1797
|
-
|
|
1798
|
-
loop().catch((err) => {
|
|
1799
|
-
console.error("push worker stopped unexpectedly:", err);
|
|
1800
|
-
this.pushWorkerStarted = false;
|
|
1801
|
-
this.startPushWorker();
|
|
1802
|
-
});
|
|
1803
|
-
}
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
// 原始推送逻辑,供队列消费
|
|
1807
|
-
private async _doPushRecord(
|
|
1808
|
-
tid: ThreadID,
|
|
1809
|
-
lid: PeerId,
|
|
1810
|
-
rec: IRecord,
|
|
1811
|
-
counter: number
|
|
1812
|
-
): Promise<void> {
|
|
1813
|
-
try {
|
|
1814
|
-
const addrs: TMultiaddr[] = [];
|
|
1815
|
-
const info = await this.logstore.getThread(tid);
|
|
1816
|
-
for (const l of info.logs) {
|
|
1817
|
-
if (l.addrs && l.addrs.length > 0) {
|
|
1818
|
-
addrs.push(...l.addrs);
|
|
1819
|
-
}
|
|
1820
|
-
}
|
|
1821
|
-
const peers = await this.getPeers(tid);
|
|
1822
|
-
if (!peers) throw new Error(`No peers for thread ${tid}`);
|
|
1823
|
-
for (const p of peers) {
|
|
1824
|
-
if (!p || p.toString() === "") continue;
|
|
1825
|
-
let dbClient: DBClient | null = null;
|
|
1826
|
-
try {
|
|
1827
|
-
const [client, _] = await this.getClient(p);
|
|
1828
|
-
if (!client) continue;
|
|
1829
|
-
dbClient = new DBClient(client, this.dc, this, this.logstore);
|
|
1830
|
-
await dbClient.pushRecordToPeer(tid, lid, rec, counter);
|
|
1831
|
-
} catch (err) {
|
|
1832
|
-
if (err instanceof Error && dbClient && err.message === Errors.ErrLogNotFound.message) { //log文件没绑定,进行绑定
|
|
1833
|
-
try {
|
|
1834
|
-
await this.context.dbManager?.addLogToThreadStart(null,tid, lid);
|
|
1835
|
-
const err = await dbClient.pushLogToPeer(tid, lid,rec); //推送本地log文件到对等节点,rec表示最新的记录
|
|
1836
|
-
if (err){
|
|
1837
|
-
console.error("Failed to push log after adding to thread:", err);
|
|
1838
|
-
}else {
|
|
1839
|
-
await dbClient.pushRecordToPeer(tid, lid, rec, counter);
|
|
1840
|
-
}
|
|
1841
|
-
}catch (err) {
|
|
1842
|
-
console.error("Failed to create transfer stream for pushing log:", err);
|
|
1843
|
-
}
|
|
1844
|
-
}
|
|
1845
|
-
}
|
|
1846
|
-
}
|
|
1847
|
-
} catch (err) {
|
|
1848
|
-
throw new Error(
|
|
1849
|
-
`Failed to push record: ${
|
|
1850
|
-
err instanceof Error ? err.message : String(err)
|
|
1851
|
-
}`
|
|
1852
|
-
);
|
|
1853
|
-
}
|
|
1854
|
-
}
|
|
1855
|
-
|
|
1856
|
-
/**
|
|
1857
|
-
* 推送记录到日志地址和threaddb 主题
|
|
1858
|
-
* @param tid threaddb ID
|
|
1859
|
-
* @param lid 对等点ID
|
|
1860
|
-
* @param rec 记录对象
|
|
1861
|
-
* @param counter 计数器
|
|
1862
|
-
*/
|
|
1863
|
-
async pushRecord(
|
|
1864
|
-
tid: ThreadID,
|
|
1865
|
-
lid: PeerId,
|
|
1866
|
-
rec: IRecord,
|
|
1867
|
-
counter: number
|
|
1868
|
-
): Promise<void> {
|
|
1869
|
-
this.pushQueue.push({ tid, lid, rec, counter });
|
|
1870
|
-
this.startPushWorker();
|
|
1871
|
-
}
|
|
1872
|
-
|
|
1873
|
-
/**
|
|
1874
|
-
* 与拥有该threaddb 的对等节点交换threaddb 记录
|
|
1875
|
-
* @param tid threaddb ID
|
|
1876
|
-
*/
|
|
1877
|
-
async exchange(tid: ThreadID): Promise<void> {
|
|
1878
|
-
try {
|
|
1879
|
-
// 获取threaddb 信息
|
|
1880
|
-
const info = await this.logstore.getThread(tid);
|
|
1881
|
-
|
|
1882
|
-
// 收集所有日志的地址
|
|
1883
|
-
const addrs: TMultiaddr[] = [];
|
|
1884
|
-
for (const lg of info.logs) {
|
|
1885
|
-
addrs.push(...lg.addrs);
|
|
1886
|
-
}
|
|
1887
|
-
|
|
1888
|
-
// 获取唯一对等点
|
|
1889
|
-
let peers = await this.uniquePeers(addrs);
|
|
1890
|
-
|
|
1891
|
-
// 尝试从外部对象获取对等点
|
|
1892
|
-
const extPeers = await this.getPeers(tid);
|
|
1893
|
-
if (extPeers && extPeers.length > 0) {
|
|
1894
|
-
peers = extPeers;
|
|
1895
|
-
}
|
|
1896
|
-
|
|
1897
|
-
// 与每个对等点交换
|
|
1898
|
-
for (const pid of peers) {
|
|
1899
|
-
// 使用异步方式处理交换,不等待完成
|
|
1900
|
-
this.exchangeWithPeer(pid, tid).catch((err) => {
|
|
1901
|
-
console.error(
|
|
1902
|
-
`Error exchanging with peer ${pid}: ${
|
|
1903
|
-
err instanceof Error ? err.message : String(err)
|
|
1904
|
-
}`
|
|
1905
|
-
);
|
|
1906
|
-
});
|
|
1907
|
-
}
|
|
1908
|
-
} catch (err) {
|
|
1909
|
-
throw new Error(
|
|
1910
|
-
`Exchange failed: ${err instanceof Error ? err.message : String(err)}`
|
|
1911
|
-
);
|
|
1912
|
-
}
|
|
1913
|
-
}
|
|
1914
|
-
|
|
1915
|
-
/**
|
|
1916
|
-
* 与单个对等点交换threaddb 边缘
|
|
1917
|
-
* @param pid 对等点ID
|
|
1918
|
-
* @param tid threaddb ID
|
|
1919
|
-
*/
|
|
1920
|
-
private async exchangeWithPeer(pid: PeerId, tid: ThreadID): Promise<void> {
|
|
1921
|
-
try {
|
|
1922
|
-
// 获取客户端连接
|
|
1923
|
-
const [client, _] = await this.getClient(pid);
|
|
1924
|
-
if (!client) {
|
|
1925
|
-
return;
|
|
1926
|
-
}
|
|
1927
|
-
|
|
1928
|
-
// 创建数据库客户端
|
|
1929
|
-
const dbClient = new DBClient(client, this.dc, this, this.logstore);
|
|
1930
|
-
|
|
1931
|
-
// 交换边缘
|
|
1932
|
-
await dbClient.exchangeEdges([tid]);
|
|
1933
|
-
} catch (err) {
|
|
1934
|
-
throw err;
|
|
1935
|
-
}
|
|
1936
|
-
}
|
|
1937
|
-
|
|
1938
|
-
/**
|
|
1939
|
-
* 获取受保护的threaddb 连接器
|
|
1940
|
-
*/
|
|
1941
|
-
getConnectorProtected(
|
|
1942
|
-
id: ThreadID,
|
|
1943
|
-
token?: Token
|
|
1944
|
-
): [Connector | undefined, boolean] {
|
|
1945
|
-
const [conn, exists] = this.getConnector(id);
|
|
1946
|
-
|
|
1947
|
-
if (!exists) {
|
|
1948
|
-
return [undefined, true]; // threaddb 未被连接器使用
|
|
1949
|
-
}
|
|
1950
|
-
|
|
1951
|
-
if (!token || !conn?.token || !this.bytesEqual(token, conn.token)) {
|
|
1952
|
-
return [undefined, false]; // 无效令牌
|
|
1953
|
-
}
|
|
1954
|
-
|
|
1955
|
-
return [conn, true];
|
|
1956
|
-
}
|
|
1957
|
-
|
|
1958
|
-
/**
|
|
1959
|
-
* 检查两个字节数组是否相等
|
|
1960
|
-
*/
|
|
1961
|
-
bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
|
|
1962
|
-
if (a.length !== b.length) return false;
|
|
1963
|
-
|
|
1964
|
-
for (let i = 0; i < a.length; i++) {
|
|
1965
|
-
if (a[i] !== b[i]) return false;
|
|
1966
|
-
}
|
|
1967
|
-
|
|
1968
|
-
return true;
|
|
1969
|
-
}
|
|
1970
|
-
}
|
|
1971
|
-
|
|
1972
|
-
/**
|
|
1973
|
-
* 记录收集器,用于合并多个对等点的记录
|
|
1974
|
-
* 线程安全版本
|
|
1975
|
-
*/
|
|
1976
|
-
class RecordCollector {
|
|
1977
|
-
private records: Map<string, Map<string, any>> = new Map();
|
|
1978
|
-
private counters: Map<string, number> = new Map();
|
|
1979
|
-
private mutex: AsyncMutex = new AsyncMutex();
|
|
1980
|
-
|
|
1981
|
-
async updateHeadCounter(logId: string, counter: number): Promise<void> {
|
|
1982
|
-
await this.mutex.acquire();
|
|
1983
|
-
try {
|
|
1984
|
-
const current = this.counters.get(logId) || 0;
|
|
1985
|
-
if (counter > current) {
|
|
1986
|
-
this.counters.set(logId, counter);
|
|
1987
|
-
}
|
|
1988
|
-
} finally {
|
|
1989
|
-
this.mutex.release();
|
|
1990
|
-
}
|
|
1991
|
-
}
|
|
1992
|
-
|
|
1993
|
-
async store(logId: string, record: any): Promise<void> {
|
|
1994
|
-
await this.mutex.acquire();
|
|
1995
|
-
try {
|
|
1996
|
-
let logRecords = this.records.get(logId);
|
|
1997
|
-
if (!logRecords) {
|
|
1998
|
-
logRecords = new Map();
|
|
1999
|
-
this.records.set(logId, logRecords);
|
|
2000
|
-
}
|
|
2001
|
-
// 使用记录的CID作为键,避免重复
|
|
2002
|
-
const key = record.cid().toString();
|
|
2003
|
-
logRecords.set(key, record);
|
|
2004
|
-
} finally {
|
|
2005
|
-
this.mutex.release();
|
|
2006
|
-
}
|
|
2007
|
-
}
|
|
2008
|
-
|
|
2009
|
-
async batchUpdate(logId: string, rs: PeerRecords): Promise<void> {
|
|
2010
|
-
await this.mutex.acquire();
|
|
2011
|
-
try {
|
|
2012
|
-
// 更新计数器
|
|
2013
|
-
const current = this.counters.get(logId) || 0;
|
|
2014
|
-
if (rs.counter > current) {
|
|
2015
|
-
this.counters.set(logId, rs.counter);
|
|
2016
|
-
}
|
|
2017
|
-
|
|
2018
|
-
// 存储记录
|
|
2019
|
-
let logRecords = this.records.get(logId);
|
|
2020
|
-
if (!logRecords) {
|
|
2021
|
-
logRecords = new Map();
|
|
2022
|
-
this.records.set(logId, logRecords);
|
|
2023
|
-
}
|
|
2024
|
-
|
|
2025
|
-
rs.records.forEach(record => {
|
|
2026
|
-
const key = record.cid().toString();
|
|
2027
|
-
logRecords!.set(key, record);
|
|
2028
|
-
});
|
|
2029
|
-
} finally {
|
|
2030
|
-
this.mutex.release();
|
|
2031
|
-
}
|
|
2032
|
-
}
|
|
2033
|
-
|
|
2034
|
-
list(): Record<string, PeerRecords> {
|
|
2035
|
-
const result: Record<string, PeerRecords> = {};
|
|
2036
|
-
|
|
2037
|
-
this.records.forEach((logRecords, logId) => {
|
|
2038
|
-
const records = Array.from(logRecords.values());
|
|
2039
|
-
result[logId] = {
|
|
2040
|
-
records,
|
|
2041
|
-
counter: this.counters.get(logId) || 0,
|
|
2042
|
-
};
|
|
2043
|
-
});
|
|
2044
|
-
|
|
2045
|
-
return result;
|
|
2046
|
-
}
|
|
2047
|
-
}
|