peerbit 4.0.7 → 4.0.8-3a75d6e

package/package.json

@@ -1,79 +1,110 @@
 {
-	"name": "peerbit",
-	"version": "4.0.7",
-	"description": "Peerbit client",
-	"author": "dao.xyz",
-	"license": "MIT",
-	"type": "module",
-	"repository": {
-		"type": "git",
-		"url": "https://github.com/dao-xyz/peerbit"
-	},
-	"engines": {
-		"node": ">=18"
-	},
-	"files": [
-		"lib",
-		"src",
-		"!src/**/__tests__",
-		"!lib/**/__tests__",
-		"!src/**/__benchmark__",
-		"!lib/**/__benchmark__",
-		"LICENSE"
-	],
-	"publishConfig": {
-		"access": "public"
-	},
-	"module": "lib/esm/index.js",
-	"types": "lib/esm/index.d.ts",
-	"exports": {
-		"import": "./lib/esm/index.js"
-	},
-	"browser": {
-		"./lib/esm/transports.js": "./lib/esm/transports-browser.js",
-		"./transports.js": "./lib/esm/transports-browser.js"
-	},
-	"dependencies": {
-		"@chainsafe/libp2p-yamux": "^6.0.1",
-		"@dao-xyz/borsh": "^5.2.1",
-		"@dao-xyz/libp2p-noise": "^14.1.2",
-		"@libp2p/circuit-relay-v2": "^1.0.13",
-		"@libp2p/identify": "^1.0.12",
-		"@libp2p/tcp": "^9.0.15",
-		"@libp2p/webrtc": "^4.0.17",
-		"@libp2p/websockets": "^8.0.15",
-		"@peerbit/any-store": "^2.0.2",
-		"@peerbit/blocks": "^2.0.32",
-		"@peerbit/crypto": "2.2.0",
-		"@peerbit/logger": "1.0.2",
-		"@peerbit/program": "4.0.7",
-		"@peerbit/pubsub": "^3.0.5",
-		"@peerbit/uint8arrays": "3.0.1",
-		"datastore-level": "^10.1.5",
-		"level": "^8.0.0",
-		"memory-level": "^1.0.0",
-		"path-browserify": "^1.0.1"
-	},
-	"devDependencies": {
-		"p-each-series": "^3.0.0",
-		"p-map-series": "^3.0.0"
-	},
-	"scripts": {
-		"clean": "shx rm -rf lib/*",
-		"build": "yarn clean && tsc -p tsconfig.json"
-	},
-	"localMaintainers": [
-		"dao.xyz"
-	],
-	"keywords": [
-		"crdt",
-		"database",
-		"distributed",
-		"decentralised",
-		"distributed",
-		"ipfs",
-		"p2p",
-		"peer-to-peer"
-	],
-	"gitHead": "fcd486b9a5c3b7d94e2ae14f1b06ab2a0f7841ad"
+  "name": "peerbit",
+  "version": "4.0.8-3a75d6e",
+  "description": "Peerbit client",
+  "author": "dao.xyz",
+  "license": "MIT",
+  "sideEffects": false,
+  "type": "module",
+  "types": "./dist/src/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "*": [
+        "*",
+        "dist/*",
+        "dist/src/*",
+        "dist/src/*/index"
+      ],
+      "src/*": [
+        "*",
+        "dist/*",
+        "dist/src/*",
+        "dist/src/*/index"
+      ]
+    }
+  },
+  "files": [
+    "src",
+    "dist",
+    "!dist/e2e",
+    "!dist/test",
+    "!**/*.tsbuildinfo"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/src/index.js"
+    }
+  },
+  "eslintConfig": {
+    "extends": "peerbit",
+    "parserOptions": {
+      "project": true,
+      "sourceType": "module"
+    },
+    "ignorePatterns": [
+      "!.aegir.js",
+      "test/ts-use",
+      "*.d.ts"
+    ]
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dao-xyz/peerbit"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "browser": {
+    "./dist/src/transports.js": "./dist/src/transports.browser.js"
+  },
+  "dependencies": {
+    "@dao-xyz/borsh": "^5.2.3",
+    "@peerbit/any-store": "2.1.0-3a75d6e",
+    "@peerbit/crypto": "2.3.0-3a75d6e",
+    "@peerbit/logger": "1.0.3-3a75d6e",
+    "@peerbit/program": "5.0.0-3a75d6e",
+    "@peerbit/pubsub": "3.0.6-3a75d6e",
+    "@peerbit/blocks": "2.1.0-3a75d6e",
+    "@peerbit/indexer-simple": "1.0.0-3a75d6e",
+    "@peerbit/indexer-sqlite3": "1.0.0-3a75d6e",
+    "datastore-level": "^10.1.5",
+    "@dao-xyz/libp2p-noise": "^14.1.2",
+    "@chainsafe/libp2p-yamux": "^6.0.2",
+    "@libp2p/webrtc": "^4.1.1",
+    "@libp2p/websockets": "^8.1.2",
+    "@libp2p/identify": "^2.1.2",
+    "@libp2p/circuit-relay-v2": "^1.1.2",
+    "@libp2p/tcp": "^9.1.1",
+    "level": "^8.0.0",
+    "memory-level": "^1.0.0",
+    "path-browserify": "^1.0.1"
+  },
+  "devDependencies": {
+    "p-each-series": "^3.0.0",
+    "p-map-series": "^3.0.0",
+    "@types/path-browserify": "^1.0.2"
+  },
+  "scripts": {
+    "clean": "aegir clean",
+    "build": "aegir build --no-bundle",
+    "test": "aegir test -t node",
+    "lint": "aegir lint"
+  },
+  "localMaintainers": [
+    "dao.xyz"
+  ],
+  "keywords": [
+    "crdt",
+    "database",
+    "distributed",
+    "decentralised",
+    "distributed",
+    "ipfs",
+    "p2p",
+    "peer-to-peer"
+  ]
 }

package/src/bootstrap.ts

@@ -1,11 +1,11 @@
 export const resolveBootstrapAddresses = async (
-	v: string = "4"
+	v: string = "4",
 ): Promise<string[]> => {
 	// Bootstrap addresses for network
 	return (
 		await (
 			await fetch(
-				`https://raw.githubusercontent.com/dao-xyz/peerbit-bootstrap/master/bootstrap${v ? "-" + v : ""}.env`
+				`https://raw.githubusercontent.com/dao-xyz/peerbit-bootstrap/master/bootstrap${v ? "-" + v : ""}.env`,
 			)
 		).text()
 	)

package/src/index.ts

@@ -4,5 +4,5 @@ export {
 	type Libp2pExtendServices,
 	type Libp2pExtended,
 	type Libp2pCreateOptions,
-	type Libp2pCreateOptionsWithServices
+	type Libp2pCreateOptionsWithServices,
 } from "./libp2p.js";

package/src/libp2p.ts

@@ -1,12 +1,17 @@
-import { createLibp2p, Libp2p, Libp2pOptions, ServiceFactoryMap } from "libp2p";
-import { DirectSub } from "@peerbit/pubsub";
-import { DirectBlock } from "@peerbit/blocks";
+import { yamux } from "@chainsafe/libp2p-yamux";
 import { noise } from "@dao-xyz/libp2p-noise";
-import { transports, relay, listen } from "./transports.js";
+import type { CircuitRelayService } from "@libp2p/circuit-relay-v2";
 import { identify } from "@libp2p/identify";
-import { CircuitRelayService } from "@libp2p/circuit-relay-v2";
-import { yamux } from "@chainsafe/libp2p-yamux";
-import { DefaultKeychain, Keychain } from "@peerbit/keychain";
+import { DirectBlock } from "@peerbit/blocks";
+import { DefaultKeychain, type Keychain } from "@peerbit/keychain";
+import { DirectSub } from "@peerbit/pubsub";
+import {
+	type Libp2p,
+	type Libp2pOptions,
+	type ServiceFactoryMap,
+	createLibp2p,
+} from "libp2p";
+import { listen, relay, transports } from "./transports.js";
 
 export type Libp2pExtendServices = {
 	pubsub: DirectSub;
@@ -18,7 +23,11 @@ export type Libp2pExtended = Libp2p<
 >;
 
 export type Libp2pCreateOptions = Libp2pOptions<
-	Libp2pExtendServices & { relay: CircuitRelayService; identify: any }
+	Partial<Libp2pExtendServices & { relay: CircuitRelayService; identify: any }>
+>;
+
+export type PartialLibp2pCreateOptions = Libp2pOptions<
+	Partial<Libp2pExtendServices & { relay: CircuitRelayService; identify: any }>
 >;
 
 export type Libp2pCreateOptionsWithServices = Libp2pCreateOptions & {
@@ -26,52 +35,53 @@ export type Libp2pCreateOptionsWithServices = Libp2pCreateOptions & {
 };
 
 export const createLibp2pExtended = (
-	opts: Libp2pCreateOptions = {
+	opts: PartialLibp2pCreateOptions = {
 		services: {
-			blocks: (c) => new DirectBlock(c),
-			pubsub: (c) => new DirectSub(c),
-			keychain: (c) => new DefaultKeychain()
-		}
-	}
+			blocks: (c: any) => new DirectBlock(c),
+			pubsub: (c: any) => new DirectSub(c),
+			keychain: () => new DefaultKeychain(),
+		},
+	},
 ): Promise<Libp2pExtended> => {
-	const relayIdentify = {
-		relay: relay(),
-		identify: identify()
-	};
+	let extraServices: any = {};
 
-	// https://github.com/libp2p/js-libp2p/issues/1757
-	Object.keys(relayIdentify).forEach((key) => {
-		if (relayIdentify[key] === undefined) {
-			delete relayIdentify[key];
+	if (!opts.services?.["relay"]) {
+		const relayComponent = relay();
+		if (relayComponent) {
+			// will be null in browser
+			extraServices["relay"] = relayComponent;
 		}
-	});
+	}
+	if (!opts.services?.["identify"]) {
+		extraServices["identify"] = identify();
+	}
 
 	return createLibp2p({
 		...opts,
 		connectionManager: {
 			minConnections: 0,
-			...opts.connectionManager
+			...opts.connectionManager,
 		},
 		addresses: {
 			listen: listen(),
-			...opts.addresses
+			...opts.addresses,
 		},
 		transports: opts.transports || transports(),
 		connectionEncryption: opts.connectionEncryption || [noise()],
 		streamMuxers: opts.streamMuxers || [yamux()],
 		services: {
-			...relayIdentify,
 			pubsub:
 				opts.services?.pubsub ||
 				((c) =>
 					new DirectSub(c, {
-						canRelayMessage: true
+						canRelayMessage: true,
 						// auto dial true
 						// auto prune true
 					})),
 			blocks: opts.services?.blocks || ((c) => new DirectBlock(c)),
 			keychain: opts.services?.keychain || ((c) => new DefaultKeychain()),
-			...opts.services
-		}
+			...opts.services,
+			...extraServices,
+		},
 	});
 };

package/src/peer.ts

@@ -1,32 +1,37 @@
-import { AnyStore } from "../../../utils/any-store/lib/esm/index.js";
-import { multiaddr, Multiaddr, isMultiaddr } from "@multiformats/multiaddr";
-import type { Libp2p } from "libp2p";
+import "@libp2p/peer-id";
+import {
+	type Multiaddr,
+	isMultiaddr,
+	multiaddr,
+} from "@multiformats/multiaddr";
+import { type AnyStore, createStore } from "@peerbit/any-store";
+import { DirectBlock } from "@peerbit/blocks";
 import { Ed25519Keypair, Ed25519PublicKey } from "@peerbit/crypto";
+import type { Indices } from "@peerbit/indexer-interface";
+import { create as createSQLiteIndexer } from "@peerbit/indexer-sqlite3";
+import { DefaultKeychain } from "@peerbit/keychain";
+import { logger as loggerFn } from "@peerbit/logger";
 import {
-	Program,
-	Address,
-	ProgramClient,
-	ProgramHandler
+	type Address,
+	type ExtractArgs,
+	type OpenOptions,
+	type Program,
+	type ProgramClient,
+	ProgramHandler,
 } from "@peerbit/program";
 import { DirectSub } from "@peerbit/pubsub";
+import { waitFor } from "@peerbit/time";
+import { LevelDatastore } from "datastore-level";
+import type { Libp2p } from "libp2p";
 import sodium from "libsodium-wrappers";
 import path from "path-browserify";
-import { waitFor } from "@peerbit/time";
-import "@libp2p/peer-id";
-
+import { resolveBootstrapAddresses } from "./bootstrap.js";
 import {
+	type Libp2pCreateOptions as ClientCreateOptions,
+	type Libp2pExtended,
+	type PartialLibp2pCreateOptions,
 	createLibp2pExtended,
-	Libp2pExtended,
-	Libp2pCreateOptions as ClientCreateOptions
 } from "./libp2p.js";
-import { DirectBlock } from "@peerbit/blocks";
-import { LevelDatastore } from "datastore-level";
-import { logger as loggerFn } from "@peerbit/logger";
-import { OpenOptions } from "@peerbit/program";
-import { resolveBootstrapAddresses } from "./bootstrap.js";
-import { createStore } from "@peerbit/any-store";
-import { DefaultKeychain } from "@peerbit/keychain";
-import { ExtractArgs } from "@peerbit/program";
 
 export const logger = loggerFn({ module: "client" });
 
@@ -36,12 +41,14 @@ export type OptionalCreateOptions = {
 export type CreateOptions = {
 	directory?: string;
 	storage: AnyStore;
+	indexer: Indices;
 	identity: Ed25519Keypair;
 } & OptionalCreateOptions;
-type Libp2pOptions = { libp2p?: Libp2pExtended | ClientCreateOptions };
+type Libp2pOptions = { libp2p?: Libp2pExtended | PartialLibp2pCreateOptions };
 type SimpleLibp2pOptions = { relay?: boolean };
 export type CreateInstanceOptions = (SimpleLibp2pOptions | Libp2pOptions) & {
 	directory?: string;
+	indexer?: (directory?: string) => Promise<Indices> | Indices;
 } & OptionalCreateOptions;
 
 const isLibp2pInstance = (libp2p: Libp2pExtended | ClientCreateOptions) =>
@@ -49,7 +56,7 @@ const isLibp2pInstance = (libp2p: Libp2pExtended | ClientCreateOptions) =>
 
 const createCache = async (
 	directory: string | undefined,
-	options?: { reset?: boolean }
+	options?: { reset?: boolean },
 ) => {
 	const cache = createStore(directory);
 
@@ -69,6 +76,7 @@ export class Peerbit implements ProgramClient {
 	directory?: string;
 
 	private _storage: AnyStore;
+	private _indexer: Indices;
 	private _libp2pExternal?: boolean = false;
 
 	// Libp2p peerid in Identity form
@@ -83,7 +91,7 @@ export class Peerbit implements ProgramClient {
 		if (this.libp2p.peerId.type !== "Ed25519") {
 			throw new Error(
 				"Unsupported id type, expecting Ed25519 but got " +
-					this.libp2p.peerId.type
+					this.libp2p.peerId.type,
 			);
 		}
 
@@ -95,6 +103,7 @@ export class Peerbit implements ProgramClient {
 		this.directory = options.directory;
 		this._storage = options.storage;
 		this._libp2pExternal = options.libp2pExternal;
+		this._indexer = options.indexer;
 	}
 
 	static async create(options: CreateInstanceOptions = {}): Promise<Peerbit> {
@@ -102,27 +111,32 @@ export class Peerbit implements ProgramClient {
 
 		let libp2pExtended: Libp2pExtended | undefined = (options as Libp2pOptions)
 			.libp2p as Libp2pExtended;
+
 		const asRelay = (options as SimpleLibp2pOptions).relay;
 
 		const directory = options.directory;
 		const hasDir = directory != null;
 
 		const storage = await createCache(
-			directory != null ? path.join(directory, "/cache") : undefined
+			directory != null ? path.join(directory, "/cache") : undefined,
 		);
 
+		const indexerFn = options.indexer || createSQLiteIndexer;
+		const indexer =
+			directory != null
+				? await indexerFn(path.join(directory, "/index"))
+				: await indexerFn();
+
 		const blocksDirectory = hasDir
-			? path.join(options["directory"], "/blocks").toString()
+			? path.join(directory, "/blocks").toString()
 			: undefined;
 
 		const keychainDirectory = hasDir
-			? path.join(options["directory"], "/keychain").toString()
+			? path.join(directory, "/keychain").toString()
 			: undefined;
 
 		const datastore = hasDir
-			? new LevelDatastore(
-					path.join(options["directory"], "/libp2p").toString()
-				)
+			? new LevelDatastore(path.join(directory, "/libp2p").toString())
 			: undefined;
 
 		if (datastore) {
@@ -133,8 +147,11 @@ export class Peerbit implements ProgramClient {
 		if (!libp2pExternal) {
 			const extendedOptions: ClientCreateOptions | undefined =
 				libp2pExtended as any as ClientCreateOptions;
+			const store = createStore(keychainDirectory);
+			await store.open();
+
 			const keychain = new DefaultKeychain({
-				store: createStore(keychainDirectory)
+				store,
 			});
 			const peerId =
 				extendedOptions?.peerId ||
@@ -145,16 +162,16 @@ export class Peerbit implements ProgramClient {
 				...extendedOptions,
 				peerId,
 				services: {
-					keychain: (c) => keychain,
-					blocks: (c) =>
+					keychain: (c: any) => keychain,
+					blocks: (c: any) =>
 						new DirectBlock(c, {
 							canRelayMessage: asRelay,
-							directory: blocksDirectory
+							directory: blocksDirectory,
 						}),
-					pubsub: (c) => new DirectSub(c, { canRelayMessage: asRelay }),
-					...extendedOptions?.services
-				},
-				datastore
+					pubsub: (c: any) => new DirectSub(c, { canRelayMessage: asRelay }),
+					...extendedOptions?.services,
+				} as any, // TODO types are funky
+				datastore,
 			});
 		}
 		if (datastore) {
@@ -175,7 +192,7 @@ export class Peerbit implements ProgramClient {
 		if (libp2pExtended.peerId.type !== "Ed25519") {
 			throw new Error(
 				"Unsupported id type, expecting Ed25519 but got " +
-					libp2pExtended.peerId.type
+					libp2pExtended.peerId.type,
 			);
 		}
 
@@ -183,10 +200,10 @@ export class Peerbit implements ProgramClient {
 		try {
 			await libp2pExtended.services.keychain.import({
 				keypair: identity,
-				id: SELF_IDENTITY_KEY_ID
+				id: SELF_IDENTITY_KEY_ID,
 			});
 		} catch (error: any) {
-			if (error.code == "ERR_KEY_ALREADY_EXISTS") {
+			if (error.code === "ERR_KEY_ALREADY_EXISTS") {
 				// Do nothing
 			} else {
 				throw error;
@@ -195,9 +212,10 @@ export class Peerbit implements ProgramClient {
 
 		const peer = new Peerbit(libp2pExtended, {
 			directory,
-			storage: storage,
+			storage,
 			libp2pExternal,
-			identity
+			identity,
+			indexer,
 		});
 		return peer;
 	}
@@ -228,10 +246,10 @@ export class Peerbit implements ProgramClient {
 	 * Dial a peer with an Ed25519 peerId
 	 */
 	async dial(
-		address: string | Multiaddr | Multiaddr[] | ProgramClient
+		address: string | Multiaddr | Multiaddr[] | ProgramClient,
 	): Promise<boolean> {
 		const maddress =
-			typeof address == "string"
+			typeof address === "string"
 				? multiaddr(address)
 				: isMultiaddr(address) || Array.isArray(address)
 					? address
@@ -244,13 +262,14 @@ export class Peerbit implements ProgramClient {
 			(await waitFor(
 				() =>
 					this.libp2p.services.pubsub.peers.has(publicKey.hashcode()) &&
-					this.libp2p.services.blocks.peers.has(publicKey.hashcode())
+					this.libp2p.services.blocks.peers.has(publicKey.hashcode()),
 			)) || false
 		);
 	}
 
 	async start() {
 		await this._storage.open();
+		await this.indexer.start();
 
 		if (this.libp2p.status === "stopped" || this.libp2p.status === "stopping") {
 			this._libp2pExternal = false; // this means we will also close libp2p client on close
@@ -260,6 +279,7 @@ export class Peerbit implements ProgramClient {
 	async stop() {
 		await this._handler?.stop();
 		await this._storage.close();
+		await this.indexer.stop();
 
 		// Close libp2p (after above)
 		if (!this._libp2pExternal) {
@@ -274,7 +294,7 @@ export class Peerbit implements ProgramClient {
 			throw new Error("Failed to find any addresses to dial");
 		}
 		const settled = await Promise.allSettled(
-			addresses.map((x) => this.dial(x))
+			addresses.map((x) => this.dial(x)),
 		);
 		let once = false;
 		for (const [i, result] of settled.entries()) {
@@ -285,7 +305,7 @@ export class Peerbit implements ProgramClient {
 					"Failed to dial bootstrap address(s): " +
 						JSON.stringify(addresses[i]) +
 						". Reason: " +
-						result.reason
+						result.reason,
 				);
 			}
 		}
@@ -304,7 +324,7 @@ export class Peerbit implements ProgramClient {
 
 	async open<S extends Program<ExtractArgs<S>>>(
 		storeOrAddress: S | Address | string,
-		options: OpenOptions<S> = {}
+		options: OpenOptions<S> = {},
 	): Promise<S> {
 		return (
 			this._handler || (this._handler = new ProgramHandler({ client: this }))
@@ -314,4 +334,8 @@ export class Peerbit implements ProgramClient {
 	get storage() {
 		return this._storage;
 	}
+
+	get indexer() {
+		return this._indexer;
+	}
 }

package/src/{transports-browser.ts → transports.browser.ts}

@@ -1,16 +1,17 @@
-import { webSockets } from "@libp2p/websockets";
 import { circuitRelayTransport } from "@libp2p/circuit-relay-v2";
 import { webRTC } from "@libp2p/webrtc";
+import { webSockets } from "@libp2p/websockets";
 import { all } from "@libp2p/websockets/filters";
 
 export const transports = () => [
 	webSockets({ filter: all }),
 	circuitRelayTransport({
-		discoverRelays: 1
+		discoverRelays: 1,
+		reservationCompletionTimeout: 5000,
 	}),
-	webRTC({})
+	webRTC({}),
 ];
 
-export const relay = () => undefined;
+export const relay = () => undefined as any;
 
 export const listen: () => string[] | undefined = () => ["/webrtc"];

package/src/transports.ts

@@ -1,17 +1,25 @@
-import { webSockets } from "@libp2p/websockets";
-import { circuitRelayTransport } from "@libp2p/circuit-relay-v2";
+import {
+	circuitRelayServer,
+	circuitRelayTransport,
+} from "@libp2p/circuit-relay-v2";
 import { tcp } from "@libp2p/tcp";
-import { circuitRelayServer } from "@libp2p/circuit-relay-v2";
+import { webSockets } from "@libp2p/websockets";
 import { all } from "@libp2p/websockets/filters";
+
 export const transports = () => [
 	webSockets({ filter: all }),
-	circuitRelayTransport(),
-	tcp()
+	circuitRelayTransport({
+		reservationCompletionTimeout: 5000,
+	}),
+	tcp(),
 ];
 export const relay = () =>
-	circuitRelayServer({ reservations: { maxReservations: 1000 } });
+	// applyDefaultLimit: false because of https://github.com/libp2p/js-libp2p/issues/2622
+	circuitRelayServer({
+		reservations: { applyDefaultLimit: false, maxReservations: 1000 },
+	});
 
 export const listen: () => string[] | undefined = () => [
 	"/ip4/127.0.0.1/tcp/0",
-	"/ip4/127.0.0.1/tcp/0/ws"
+	"/ip4/127.0.0.1/tcp/0/ws",
 ];