smoldot 0.7.13 → 1.0.1

package/dist/cjs/instance/bindings-smoldot-light.d.ts

@@ -63,12 +63,33 @@ export interface Connection {
     /**
      * Queues data to be sent on the given connection.
      *
-     * The connection must currently be in the `Open` state.
+     * The connection and stream must currently be in the `Open` state.
+     *
+     * The number of bytes most never exceed the number of "writable bytes" of the stream.
+     * `onWritableBytes` can be used in order to notify that more writable bytes are available.
      *
      * The `streamId` must be provided if and only if the connection is of type "multi-stream".
      * It indicates which substream to send the data on.
+     *
+     * Must not be called after `closeSend` has been called.
      */
     send(data: Uint8Array, streamId?: number): void;
+    /**
+     * Closes the writing side of the given stream of the given connection.
+     *
+     * Never called for connection types where this isn't possible to implement (i.e. WebSocket
+     * and WebRTC at the moment).
+     *
+     * The connection and stream must currently be in the `Open` state.
+     *
+     * Implicitly sets the "writable bytes" of the stream to zero.
+     *
+     * The `streamId` must be provided if and only if the connection is of type "multi-stream".
+     * It indicates which substream to send the data on.
+     *
+     * Must only be called once per stream.
+     */
+    closeSend(streamId?: number): void;
     /**
      * Start opening an additional outbound substream on the given connection.
      *
@@ -104,6 +125,8 @@ export interface ConnectionConfig {
     onOpen: (info: {
         type: 'single-stream';
         handshake: 'multistream-select-noise-yamux';
+        initialWritableBytes: number;
+        writeClosable: boolean;
     } | {
         type: 'multi-stream';
         handshake: 'webrtc';
@@ -121,7 +144,7 @@ export interface ConnectionConfig {
      *
      * This function must only be called for connections of type "multi-stream".
      */
-    onStreamOpened: (streamId: number, direction: 'inbound' | 'outbound') => void;
+    onStreamOpened: (streamId: number, direction: 'inbound' | 'outbound', initialWritableBytes: number) => void;
     /**
      * Callback called when a stream transitions to the `Reset` state.
      *
@@ -130,6 +153,17 @@ export interface ConnectionConfig {
      * This function must only be called for connections of type "multi-stream".
      */
     onStreamReset: (streamId: number) => void;
+    /**
+     * Callback called when more data can be written on the stream.
+     *
+     * Can only happen while the connection is in the `Open` state.
+     *
+     * This callback must not be called after `closeSend` has been called.
+     *
+     * The `streamId` parameter must be provided if and only if the connection is of type
+     * "multi-stream".
+     */
+    onWritableBytes: (numExtra: number, streamId?: number) => void;
     /**
      * Callback called when a message sent by the remote has been received.
      *

package/dist/cjs/instance/bindings-smoldot-light.js

@@ -97,7 +97,7 @@ function default_1(config) {
             // In browsers, `setTimeout` works as expected when `ms` equals 0. However, NodeJS
             // requires a minimum of 1 millisecond (if `0` is passed, it is automatically replaced
             // with `1`) and wants you to use `setImmediate` instead.
-            if (ms == 0 && typeof setImmediate === "function") {
+            if (ms < 1 && typeof setImmediate === "function") {
                 setImmediate(() => {
                     if (killedTracked.killed)
                         return;
@@ -140,7 +140,7 @@ function default_1(config) {
                         try {
                             switch (info.type) {
                                 case 'single-stream': {
-                                    instance.exports.connection_open_single_stream(connectionId, 0);
+                                    instance.exports.connection_open_single_stream(connectionId, 0, info.initialWritableBytes, info.writeClosable ? 1 : 0);
                                     break;
                                 }
                                 case 'multi-stream': {
@@ -168,6 +168,14 @@ function default_1(config) {
                         }
                         catch (_error) { }
                     },
+                    onWritableBytes: (numExtra, streamId) => {
+                        if (killedTracked.killed)
+                            return;
+                        try {
+                            instance.exports.stream_writable_bytes(connectionId, streamId || 0, numExtra);
+                        }
+                        catch (_error) { }
+                    },
                     onMessage: (message, streamId) => {
                         if (killedTracked.killed)
                             return;
@@ -178,11 +186,11 @@ function default_1(config) {
                         }
                         catch (_error) { }
                     },
-                    onStreamOpened: (streamId, direction) => {
+                    onStreamOpened: (streamId, direction, initialWritableBytes) => {
                         if (killedTracked.killed)
                             return;
                         try {
-                            instance.exports.connection_stream_opened(connectionId, streamId, direction === 'outbound' ? 1 : 0);
+                            instance.exports.connection_stream_opened(connectionId, streamId, direction === 'outbound' ? 1 : 0, initialWritableBytes);
                         }
                         catch (_error) { }
                     },
@@ -244,6 +252,12 @@ function default_1(config) {
             const connection = connections[connectionId];
             connection.send(data, streamId); // TODO: docs says the streamId is provided only for multi-stream connections, but here it's always provided
         },
+        stream_send_close: (connectionId, streamId) => {
+            if (killedTracked.killed)
+                return;
+            const connection = connections[connectionId];
+            connection.closeSend(streamId); // TODO: docs says the streamId is provided only for multi-stream connections, but here it's always provided
+        },
         current_task_entered: (ptr, len) => {
             if (killedTracked.killed)
                 return;

package/dist/cjs/instance/bindings.d.ts

@@ -19,10 +19,11 @@ export interface SmoldotWasmExports extends WebAssembly.Exports {
     json_rpc_responses_peek: (chainId: number) => number;
     json_rpc_responses_pop: (chainId: number) => void;
     timer_finished: (timerId: number) => void;
-    connection_open_single_stream: (connectionId: number, handshakeTy: number) => void;
+    connection_open_single_stream: (connectionId: number, handshakeTy: number, initialWritableBytes: number, writeClosable: number) => void;
     connection_open_multi_stream: (connectionId: number, handshakeTyPtr: number, handshakeTyLen: number) => void;
+    stream_writable_bytes: (connectionId: number, streamId: number, numBytes: number) => void;
     stream_message: (connectionId: number, streamId: number, ptr: number, len: number) => void;
-    connection_stream_opened: (connectionId: number, streamId: number, outbound: number) => void;
+    connection_stream_opened: (connectionId: number, streamId: number, outbound: number, initialWritableBytes: number) => void;
     connection_reset: (connectionId: number, ptr: number, len: number) => void;
     stream_reset: (connectionId: number, streamId: number) => void;
 }

package/dist/mjs/client.d.ts

@@ -26,17 +26,23 @@ export interface Client {
     /**
      * Connects to a chain.
      *
-     * Throws an exception if the chain specification isn't valid, or if the chain specification
-     * concerns a parachain but no corresponding relay chain can be found.
+     * After you've called this function, the client will verify whether the chain specification is
+     * valid. Once this is done, the `Promise` returned by this function will yield a
+     * {@link Chain} that can be used to interact with that chain. Only after the `Promise` has
+     * yielded will the client actually start establishing networking connections to the chain.
+     *
+     * The `Promise` throws an exception if the chain specification isn't valid, or if the chain
+     * specification concerns a parachain but no corresponding relay chain can be found.
      *
      * Smoldot will automatically de-duplicate chains if multiple identical chains are added, in
      * order to save resources. In other words, it is not a problem to call `addChain` multiple
-     * times with the same chain specifications and obtain multiple `Chain`.
+     * times with the same chain specifications and obtain multiple {@link Chain} objects.
      * When the same client is used for multiple different purposes, you are in fact strongly
      * encouraged to trust smoldot and not attempt to de-duplicate chains yourself, as determining
      * whether two chains are identical is complicated and might have security implications.
      *
-     * Smoldot tries to distribute CPU resources equally between all active `Chain` objects.
+     * Smoldot tries to distribute CPU resources equally between all active {@link Chain} objects
+     * of the same client.
      *
      * @param options Configuration of the chain to add.
      *
@@ -48,6 +54,9 @@ export interface Client {
     /**
      * Terminates the client.
      *
+     * This implicitly calls {@link Chain.remove} on all the chains associated with this client,
+     * then shuts down the client itself.
+     *
      * Afterwards, trying to use the client or any of its chains again will lead to an exception
      * being thrown.
      *
@@ -70,10 +79,15 @@ export interface Chain {
      * Be aware that some requests will cause notifications to be sent back using the same callback
      * as the responses.
      *
-     * A {@link MalformedJsonRpcError} is thrown if the request isn't a valid JSON-RPC request or
-     * if the request is unreasonably large (64 MiB at the time of writing of this comment).
-     * If, however, the request is a valid JSON-RPC request but that concerns an unknown method, a
-     * error response is properly generated.
+     * A {@link MalformedJsonRpcError} is thrown if the request isn't a valid JSON-RPC request
+     * (for example if it is not valid JSON) or if the request is unreasonably large (64 MiB at the
+     * time of writing of this comment).
+     * If, however, the request is a valid JSON-RPC request but that concerns an unknown method, or
+     * if for example some parameters are missing, an error response is properly generated and
+     * yielded through the JSON-RPC callback.
+     * In other words, a {@link MalformedJsonRpcError} is thrown in situations where something
+     * is *so wrong* with the request that it is not possible for smoldot to send back an error
+     * through the JSON-RPC callback.
      *
      * Two JSON-RPC APIs are supported by smoldot:
      *
@@ -92,14 +106,17 @@ export interface Chain {
     /**
      * Waits for a JSON-RPC response or notification to be generated.
      *
-     * If this function is called multiple times "simultaneously" (generating multiple different
-     * `Promise`s), each `Promise` will return a different JSON-RPC response or notification.
-     *
      * Each chain contains a buffer of the responses waiting to be sent out. Calling this function
      * pulls one element from the buffer. If this function is called at a slower rate than responses
-     * are generated, then buffer will eventually become full, at which point calling
+     * are generated, then the buffer will eventually become full, at which point calling
      * {@link Chain.sendJsonRpc} will throw an exception.
      *
+     * If this function is called multiple times "simultaneously" (generating multiple different
+     * `Promise`s), each `Promise` will return a different JSON-RPC response or notification. In
+     * that situation, there is no guarantee in the ordering in which the responses or notifications
+     * are yielded. Calling this function multiple times "simultaneously" is in general a niche
+     * corner case that you are encouraged to avoid.
+     *
      * @throws {@link AlreadyDestroyedError} If the chain has been removed or the client has been terminated.
      * @throws {@link JsonRpcDisabledError} If the JSON-RPC system was disabled in the options of the chain.
      * @throws {@link CrashError} If the background client has crashed.
@@ -108,14 +125,15 @@ export interface Chain {
     /**
      * Disconnects from the blockchain.
      *
-     * The JSON-RPC callback will no longer be called.
+     * The JSON-RPC callback will no longer be called. This is the case immediately after this
+     * function is called. Any on-going JSON-RPC request is instantaneously aborted.
      *
      * Trying to use the chain again will lead to an exception being thrown.
      *
      * If this chain is a relay chain, then all parachains that use it will continue to work. Smoldot
      * automatically keeps alive all relay chains that have an active parachains. There is no need
-     * to track parachains and relaychains, or to destroy them in the correct order, as this is
-     * handled automatically.
+     * to track parachains and relay chains, or to destroy them in the correct order, as this is
+     * handled automatically internally.
      *
      * @throws {@link AlreadyDestroyedError} If the chain has already been removed or the client has been terminated.
      * @throws {@link CrashError} If the background client has crashed.
@@ -140,10 +158,10 @@ export interface ClientOptions {
      */
     logCallback?: LogCallback;
     /**
-     * The client will never call the callback with a value of `level` superior to this value.
+     * The client will never call the log callback with a value of `level` superior to this value.
      * Defaults to 3.
      *
-     * While this filtering could be done directly by the `logCallback`, passing a maximum log level
+     * While this filtering could be done manually in the `logCallback`, passing a maximum log level
      * leads to better performances as the client doesn't even need to generate a `message` when it
      * knows that this message isn't interesting.
      */
@@ -226,32 +244,47 @@ export interface AddChainOptions {
      * `<client> build-spec --raw > spec.json`. Only "raw" chain specifications are supported by
      * smoldot at the moment.
      *
-     * If the chain specification contains a `relay_chain` field, then smoldot will try to match
-     * the value in `relay_chain` with the value in `id` of the chains in `potentialRelayChains`.
+     * If the chain specification contains a `relayChain` field, then smoldot will try to match
+     * the value in `relayChain` with the value in `id` of the chains in
+     * {@link AddChainOptions.potentialRelayChains}.
      */
     chainSpec: string;
     /**
-     * Content of the database of this chain. Can be obtained by using the
-     * `chainHead_unstable_finalizedDatabase` JSON-RPC function.
+     * Content of the database of this chain.
+     *
+     * The content of the database can be obtained by using the
+     * `chainHead_unstable_finalizedDatabase` JSON-RPC function. This undocumented JSON-RPC function
+     * accepts one parameter of type `number` indicating an upper limit to the size of the database.
+     * The content of the database is always a UTF-8 string whose content is at the discretion of
+     * the smoldot implementation.
      *
      * Smoldot reserves the right to change its database format, making previous databases
-     * incompatible. For this reason, no error is generated if the content of the database is invalid
-     * and/or can't be decoded.
+     * incompatible. For this reason, no error is generated if the content of the database is
+     * invalid and/or can't be decoded.
+     *
+     * Providing a database can considerably improve the time it takes for smoldot to be fully
+     * synchronized with a chain by reducing the amount of data that it has to download.
+     * Furthermore, the database also contains a list of nodes that smoldot can use in order to
+     * reduce the load that is being put on the bootnodes.
      *
      * Important: please note that using a malicious database content can lead to a security
      * vulnerability. This database content is considered by smoldot as trusted input. It is the
      * responsibility of the API user to make sure that the value passed in this field comes from
      * the same source of trust as the chain specification that was used when retrieving this
-     * database content.
+     * database content. In other words, if you load this database content for example from the disk
+     * or from the browser's local storage, be absolutely certain that no malicious program has
+     * modified the content of that file or local storage.
      */
     databaseContent?: string;
     /**
      * If `chainSpec` concerns a parachain, contains the list of chains whose `id` smoldot will try
-     * to match with the parachain's `relay_chain`.
+     * to match with the parachain's `relayChain`.
      * Defaults to `[]`.
      *
-     * Must contain exactly the objects that were returned by previous calls to `addChain`. The
-     * library uses a `WeakMap` in its implementation in order to identify chains.
+     * Must contain exactly the {@link Chain} objects that were returned by previous calls to
+     * `addChain`. The library uses a `WeakMap` in its implementation in order to identify chains.
+     *
+     * # Explanation and usage
      *
      * The primary way smoldot determines which relay chain is associated to a parachain is by
      * inspecting the chain specification of that parachain (i.e. the `chainSpec` field).
@@ -260,7 +293,7 @@ export interface AddChainOptions {
      * applications: multiple applications could add mutiple different chains with the same `id`,
      * creating an ambiguity, or an application could register malicious chains with small variations
      * of a popular chain's `id` and try to benefit from a typo in a legitimate application's
-     * `relay_chain`.
+     * `relayChain`.
      *
      * These problems can be solved by using this parameter to segregate multiple different uses of
      * the same client. To use it, pass the list of all chains that the same application has

package/dist/mjs/index-browser.js

@@ -40,6 +40,8 @@ export function start(options) {
             return Promise.resolve(inflate(classicDecode(input)));
         },
         registerShouldPeriodicallyYield: (callback) => {
+            if (typeof document === 'undefined') // We might be in a web worker.
+                return [false, () => { }];
             const wrappedCallback = () => callback(document.visibilityState === 'visible');
             document.addEventListener('visibilitychange', wrappedCallback);
             return [document.visibilityState === 'visible', () => { document.removeEventListener('visibilitychange', wrappedCallback); }];
@@ -68,9 +70,8 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
     // Attempt to parse the multiaddress.
     // TODO: remove support for `/wss` in a long time (https://github.com/paritytech/smoldot/issues/1940)
     const wsParsed = config.address.match(/^\/(ip4|ip6|dns4|dns6|dns)\/(.*?)\/tcp\/(.*?)\/(ws|wss|tls\/ws)$/);
-    const webRTCParsed = config.address.match(/^\/(ip4|ip6)\/(.*?)\/udp\/(.*?)\/webrtc\/certhash\/(.*?)$/);
+    const webRTCParsed = config.address.match(/^\/(ip4|ip6)\/(.*?)\/udp\/(.*?)\/webrtc-direct\/certhash\/(.*?)$/);
     if (wsParsed != null) {
-        let connection;
         const proto = (wsParsed[4] == 'ws') ? 'ws' : 'wss';
         if ((proto == 'ws' && forbidWs) ||
             (proto == 'ws' && wsParsed[2] != 'localhost' && wsParsed[2] != '127.0.0.1' && forbidNonLocalWs) ||
@@ -80,10 +81,36 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
         const url = (wsParsed[1] == 'ip6') ?
             (proto + "://[" + wsParsed[2] + "]:" + wsParsed[3]) :
             (proto + "://" + wsParsed[2] + ":" + wsParsed[3]);
-        connection = new WebSocket(url);
+        const connection = new WebSocket(url);
         connection.binaryType = 'arraybuffer';
+        const bufferedAmountCheck = { quenedUnreportedBytes: 0, nextTimeout: 10 };
+        const checkBufferedAmount = () => {
+            if (connection.readyState != 1)
+                return;
+            // Note that we might expect `bufferedAmount` to always be <= the sum of the lengths
+            // of all the data that has been sent, but that might not be the case. For this
+            // reason, we use `bufferedAmount` as a hint rather than a correct value.
+            const bufferedAmount = connection.bufferedAmount;
+            let wasSent = bufferedAmountCheck.quenedUnreportedBytes - bufferedAmount;
+            if (wasSent < 0)
+                wasSent = 0;
+            bufferedAmountCheck.quenedUnreportedBytes -= wasSent;
+            if (bufferedAmountCheck.quenedUnreportedBytes != 0) {
+                setTimeout(checkBufferedAmount, bufferedAmountCheck.nextTimeout);
+                bufferedAmountCheck.nextTimeout *= 2;
+                if (bufferedAmountCheck.nextTimeout > 500)
+                    bufferedAmountCheck.nextTimeout = 500;
+            }
+            // Note: it is important to call `onWritableBytes` at the very end, as it might
+            // trigger a call to `send`.
+            if (wasSent != 0)
+                config.onWritableBytes(wasSent);
+        };
         connection.onopen = () => {
-            config.onOpen({ type: 'single-stream', handshake: 'multistream-select-noise-yamux' });
+            config.onOpen({
+                type: 'single-stream', handshake: 'multistream-select-noise-yamux',
+                initialWritableBytes: 1024 * 1024, writeClosable: false,
+            });
         };
         connection.onclose = (event) => {
             const message = "Error code " + event.code + (!!event.reason ? (": " + event.reason) : "");
@@ -102,7 +129,13 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
             },
             send: (data) => {
                 connection.send(data);
+                if (bufferedAmountCheck.quenedUnreportedBytes == 0) {
+                    bufferedAmountCheck.nextTimeout = 10;
+                    setTimeout(checkBufferedAmount, 10);
+                }
+                bufferedAmountCheck.quenedUnreportedBytes += data.length;
             },
+            closeSend: () => { throw new Error('Wrong connection type'); },
             openOutSubstream: () => { throw new Error('Wrong connection type'); }
         };
     }
@@ -152,16 +185,18 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
             pc.onnegotiationneeded = null;
             pc.ondatachannel = null;
             for (const channel of Array.from(dataChannels.values())) {
-                channel.onopen = null;
-                channel.onerror = null;
-                channel.onclose = null;
-                channel.onmessage = null;
+                channel.channel.onopen = null;
+                channel.channel.onerror = null;
+                channel.channel.onclose = null;
+                channel.channel.onbufferedamountlow = null;
+                channel.channel.onmessage = null;
             }
             dataChannels.clear();
             if (handshakeDataChannel) {
                 handshakeDataChannel.onopen = null;
                 handshakeDataChannel.onerror = null;
                 handshakeDataChannel.onclose = null;
+                handshakeDataChannel.onbufferedamountlow = null;
                 handshakeDataChannel.onmessage = null;
             }
             handshakeDataChannel = undefined;
@@ -190,7 +225,7 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
                 }
                 else {
                     console.assert(direction !== 'outbound' || !handshakeDataChannel, "handshakeDataChannel still defined");
-                    config.onStreamOpened(dataChannelId, direction);
+                    config.onStreamOpened(dataChannelId, direction, 65536);
                 }
             };
             dataChannel.onerror = dataChannel.onclose = (_error) => {
@@ -212,6 +247,7 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
                     handshakeDataChannel.onopen = null;
                     handshakeDataChannel.onerror = null;
                     handshakeDataChannel.onclose = null;
+                    handshakeDataChannel.onbufferedamountlow = null;
                     handshakeDataChannel.onmessage = null;
                     handshakeDataChannel = undefined;
                 }
@@ -228,12 +264,18 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
                     config.onStreamReset(dataChannelId);
                 }
             };
+            dataChannel.onbufferedamountlow = () => {
+                const channel = dataChannels.get(dataChannelId);
+                const val = channel.bufferedBytes;
+                channel.bufferedBytes = 0;
+                config.onWritableBytes(val, dataChannelId);
+            };
             dataChannel.onmessage = (m) => {
                 // The `data` field is an `ArrayBuffer`.
                 config.onMessage(new Uint8Array(m.data), dataChannelId);
             };
             if (direction !== 'first-outbound')
-                dataChannels.set(dataChannelId, dataChannel);
+                dataChannels.set(dataChannelId, { channel: dataChannel, bufferedBytes: 0 });
             else
                 handshakeDataChannel = dataChannel;
         };
@@ -375,7 +417,8 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
                     // (UDP or TCP)
                     "a=sctp-port:5000" + "\n" +
                     // The maximum SCTP user message size (in bytes) (RFC8841)
-                    "a=max-message-size:16384" + "\n" + // TODO: should this be part of the spec?
+                    // Setting this field is part of the libp2p spec.
+                    "a=max-message-size:16384" + "\n" +
                     // A transport address for a candidate that can be used for connectivity
                     // checks (RFC8839).
                     "a=candidate:1 1 UDP 1 " + targetIp + " " + targetPort + " typ host" + "\n";
@@ -404,17 +447,21 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
                 }
                 else {
                     const channel = dataChannels.get(streamId);
-                    channel.onopen = null;
-                    channel.onerror = null;
-                    channel.onclose = null;
-                    channel.onmessage = null;
-                    channel.close();
+                    channel.channel.onopen = null;
+                    channel.channel.onerror = null;
+                    channel.channel.onclose = null;
+                    channel.channel.onbufferedamountlow = null;
+                    channel.channel.onmessage = null;
+                    channel.channel.close();
                     dataChannels.delete(streamId);
                 }
             },
             send: (data, streamId) => {
-                dataChannels.get(streamId).send(data);
+                const channel = dataChannels.get(streamId);
+                channel.channel.send(data);
+                channel.bufferedBytes += data.length;
             },
+            closeSend: () => { throw new Error('Wrong connection type'); },
             openOutSubstream: () => {
                 // `openOutSubstream` can only be called after we have called `config.onOpen`, therefore
                 // `pc` is guaranteed to be non-null.
@@ -426,8 +473,8 @@ function connect(config, forbidWs, forbidNonLocalWs, forbidWss, forbidWebRTC) {
                         // We need to check again if `handshakeDataChannel` is still defined, as the
                         // connection might have been closed.
                         if (handshakeDataChannel) {
-                            config.onStreamOpened(handshakeDataChannel.id, 'outbound');
-                            dataChannels.set(handshakeDataChannel.id, handshakeDataChannel);
+                            config.onStreamOpened(handshakeDataChannel.id, 'outbound', 1024 * 1024);
+                            dataChannels.set(handshakeDataChannel.id, { channel: handshakeDataChannel, bufferedBytes: 0 });
                             handshakeDataChannel = undefined;
                         }
                     }))();