seyfert 2.0.0 → 2.1.1-dev-11310514874.0

package/lib/websocket/discord/sharder.js

@@ -53,10 +53,10 @@ class ShardManager extends Map {
         return acc / this.size;
     }
     calculateShardId(guildId) {
-        return Number((BigInt(guildId) >> 22n) % BigInt(this.options.info.shards ?? 1));
+        return (0, common_1.calculateShardId)(guildId, this.totalShards);
     }
-    spawn(shardId) {
-        this.debugger?.info(`Spawn shard ${shardId}`);
+    create(shardId) {
+        this.debugger?.info(`Creating shard ${shardId}`);
         let shard = this.get(shardId);
         shard ??= new shard_1.Shard(shardId, {
             token: this.options.token,
@@ -73,7 +73,7 @@ class ShardManager extends Map {
     }
     async spawnShards() {
         const buckets = this.spawnBuckets();
-        this.debugger?.info('Spawn shards');
+        this.debugger?.info('Spawning shards');
         for (const bucket of buckets) {
             for (const shard of bucket) {
                 if (!shard) {
@@ -83,6 +83,81 @@ class ShardManager extends Map {
                 this.connectQueue.push(shard.connect.bind(shard));
             }
         }
+        await this.startResharder();
+    }
+    async startResharder() {
+        if (this.options.resharding.interval <= 0)
+            return;
+        if (this.shardStart !== 0 || this.shardEnd !== this.totalShards)
+            return this.debugger?.debug('Cannot start resharder');
+        this.debugger?.debug('Resharder enabled');
+        setInterval(async () => {
+            this.debugger?.debug('Checking if reshard is needed');
+            const info = await this.options.resharding.getInfo();
+            if (info.shards <= this.totalShards)
+                return this.debugger?.debug('Resharding not needed');
+            //https://github.com/discordeno/discordeno/blob/6a5f446c0651b9fad9f1550ff1857fe7a026426b/packages/gateway/src/manager.ts#L106C8-L106C94
+            const percentage = (info.shards / ((this.totalShards * 2500) / 1000)) * 100;
+            if (percentage < this.options.resharding.percentage)
+                return this.debugger?.debug(`Percentage is not enough to reshard ${percentage}/${this.options.resharding.percentage}`);
+            this.debugger?.info('Starting resharding process');
+            this.connectQueue.concurrency = info.session_start_limit.max_concurrency;
+            this.options.info.session_start_limit.max_concurrency = info.session_start_limit.max_concurrency;
+            //waiting for all shards to connect
+            let shardsConnected = 0;
+            const handleGuilds = new Set();
+            let handlePayload = async (sharder, _, packet) => {
+                if (packet.t === 'GUILD_CREATE' || packet.t === 'GUILD_DELETE') {
+                    handleGuilds.delete(packet.d.id);
+                    if (shardsConnected === info.shards && !handleGuilds.size) {
+                        return cleanProcess(sharder);
+                    }
+                }
+                if (packet.t !== 'READY')
+                    return;
+                for (const guild of packet.d.guilds) {
+                    handleGuilds.add(guild.id);
+                }
+                if (++shardsConnected < info.shards || handleGuilds.size)
+                    return;
+                cleanProcess(sharder);
+                // dont listen more events when all shards are ready
+            };
+            const cleanProcess = (sharder) => {
+                handlePayload = async () => {
+                    //
+                };
+                this.disconnectAll();
+                this.clear();
+                this.options.totalShards = this.options.shardEnd = info.shards;
+                for (const [id, shard] of sharder) {
+                    shard.options.handlePayload = (shardId, packet) => {
+                        return this.options.handlePayload(shardId, packet);
+                    };
+                    this.set(id, shard);
+                }
+                sharder.clear();
+            };
+            const options = (0, common_1.MergeOptions)(this.options, {
+                totalShards: info.shards,
+                shardEnd: info.shards,
+            });
+            const resharder = new ShardManager({
+                ...options,
+                resharding: {
+                    // getInfo mock, we don't need it
+                    getInfo: () => ({}),
+                    interval: 0,
+                    percentage: 0,
+                },
+                handlePayload: (shardId, packet) => {
+                    return handlePayload(resharder, shardId, packet);
+                },
+            });
+            // share ratelimit
+            resharder.connectQueue = this.connectQueue;
+            await resharder.spawnShards();
+        }, this.options.resharding.interval);
     }
     /*
      * spawns buckets in order
@@ -94,7 +169,7 @@ class ShardManager extends Map {
         chunks.forEach((arr, index) => {
             for (let i = 0; i < arr.length; i++) {
                 const id = i + (index > 0 ? index * this.concurrency : 0) + this.shardStart;
-                chunks[index][i] = this.spawn(id);
+                chunks[index][i] = this.create(id);
             }
         });
         this.debugger?.info(`${chunks.length} buckets created`);
@@ -102,7 +177,7 @@ class ShardManager extends Map {
     }
     forceIdentify(shardId) {
         this.debugger?.info(`Shard #${shardId} force identify`);
-        return this.spawn(shardId).identify();
+        return this.create(shardId).identify();
     }
     disconnect(shardId) {
         this.debugger?.info(`Shard #${shardId} force disconnect`);
@@ -110,10 +185,7 @@ class ShardManager extends Map {
     }
     disconnectAll() {
         this.debugger?.info('Disconnect all shards');
-        return new Promise(resolve => {
-            this.forEach(shard => shard.disconnect());
-            resolve(null);
-        });
+        this.forEach(shard => shard.disconnect());
     }
     setShardPresence(shardId, payload) {
         this.debugger?.info(`Shard #${shardId} update presence`);
@@ -123,12 +195,9 @@ class ShardManager extends Map {
         });
     }
     setPresence(payload) {
-        return new Promise(resolve => {
-            this.forEach(shard => {
-                this.setShardPresence(shard.id, payload);
-            }, this);
-            resolve();
-        });
+        this.forEach(shard => {
+            this.setShardPresence(shard.id, payload);
+        }, this);
     }
     joinVoice(guild_id, channel_id, options) {
         const shardId = this.calculateShardId(guild_id);
@@ -144,7 +213,6 @@ class ShardManager extends Map {
     }
     leaveVoice(guild_id) {
         const shardId = this.calculateShardId(guild_id);
-        this.debugger?.info(`Shard #${shardId} leave voice in ${guild_id}`);
         return this.send(shardId, {
             op: types_1.GatewayOpcodes.VoiceStateUpdate,
             d: {
@@ -165,5 +233,12 @@ class ShardManager extends Map {
         }
         this.get(shardId)?.send(false, payload);
     }
+    resume(shardId, shardData) {
+        if (this.has(shardId))
+            throw new Error('Cannot override existing shard');
+        const shard = this.create(shardId);
+        shard.data = shardData;
+        return this.connectQueue.push(shard.connect.bind(shard));
+    }
 }
 exports.ShardManager = ShardManager;

package/lib/websocket/discord/shared.d.ts

@@ -1,5 +1,5 @@
-import type { APIGatewayBotInfo, GatewayDispatchPayload, GatewayIntentBits, GatewayPresenceUpdateData } from '../../types';
 import type { Awaitable, DeepPartial, Logger } from '../../common';
+import type { APIGatewayBotInfo, GatewayDispatchPayload, GatewayIntentBits, GatewayPresenceUpdateData } from '../../types';
 import type { IdentifyProperties } from '../constants';
 export interface ShardManagerOptions extends ShardDetails {
     /** Important data which is used by the manager to connect shards to the gateway. */
@@ -29,6 +29,14 @@ export interface ShardManagerOptions extends ShardDetails {
      */
     presence?: (shardId: number, workerId: number) => GatewayPresenceUpdateData;
     compress?: boolean;
+    resharding?: {
+        /**
+         * @returns the gateway connection info
+         */
+        getInfo(): Promise<APIGatewayBotInfo>;
+        interval: number;
+        percentage: number;
+    };
 }
 export interface CustomManagerAdapter {
     postMessage(workerId: number, body: unknown): Awaitable<unknown>;
@@ -39,12 +47,12 @@ export interface WorkerManagerOptions extends Omit<ShardManagerOptions, 'handleP
     adapter?: CustomManagerAdapter;
     workers?: number;
     /**
-     * @default 32
+     * @default 16
      */
     shardsPerWorker?: number;
     workerProxy?: boolean;
     path: string;
-    handlePayload(shardId: number, workerId: number, packet: GatewayDispatchPayload): unknown;
+    handlePayload?(shardId: number, workerId: number, packet: GatewayDispatchPayload): any;
     properties?: DeepPartial<NonNullable<ShardManagerOptions['properties']>>;
 }
 export interface ShardData {
@@ -109,4 +117,5 @@ export interface WorkerData {
     debug: boolean;
     workerProxy: boolean;
     __USING_WATCHER__?: boolean;
+    resharding: boolean;
 }

package/lib/websocket/discord/socket/custom.d.ts

@@ -19,7 +19,7 @@ export declare class SeyfertWebSocket {
     private connect;
     handleReadable(): void;
     handleEvent(body: Buffer, opcode: number): void;
-    handleClose(): void;
+    handleClose(): Promise<void>;
     send(data: string): void;
     private _write;
     onping(_data: string): void;

package/lib/websocket/discord/socket/custom.js

@@ -18,38 +18,51 @@ class SeyfertWebSocket {
         this.path = `${urlParts.pathname}${urlParts.search || ''}`;
         this.connect();
     }
-    connect() {
-        const key = (0, node_crypto_1.randomBytes)(16).toString('base64');
-        const req = (0, node_https_1.request)({
-            //discord gateway hostname
-            hostname: this.hostname,
-            path: this.path,
-            headers: {
-                Connection: 'Upgrade',
-                Upgrade: 'websocket',
-                'Sec-WebSocket-Key': key,
-                'Sec-WebSocket-Version': '13',
-            },
-        });
-        req.on('upgrade', (res, socket) => {
-            const hash = (0, node_crypto_1.createHash)('sha1').update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`).digest('base64');
-            const accept = res.headers['sec-websocket-accept'];
-            if (accept !== hash) {
-                socket.end(() => {
-                    this.onerror(new Error('Invalid sec-websocket-accept header'));
-                });
-                return;
-            }
-            this.socket = socket;
-            socket.on('readable', this.handleReadable.bind(this));
-            socket.on('close', this.handleClose.bind(this));
-            socket.on('error', err => this.onerror(err));
-            this.onopen();
-        });
-        req.on('close', () => {
-            req.removeAllListeners();
+    connect(retries = 0) {
+        return new Promise((resolve, rej) => {
+            const key = (0, node_crypto_1.randomBytes)(16).toString('base64');
+            const req = (0, node_https_1.request)({
+                //discord gateway hostname
+                hostname: this.hostname,
+                path: this.path,
+                headers: {
+                    Connection: 'Upgrade',
+                    Upgrade: 'websocket',
+                    'Sec-WebSocket-Key': key,
+                    'Sec-WebSocket-Version': '13',
+                },
+            });
+            req.on('upgrade', (res, socket) => {
+                const hash = (0, node_crypto_1.createHash)('sha1').update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`).digest('base64');
+                const accept = res.headers['sec-websocket-accept'];
+                if (accept !== hash) {
+                    socket.end(() => {
+                        rej(new Error('Invalid sec-websocket-accept header'));
+                    });
+                    return;
+                }
+                this.socket = socket;
+                socket.on('readable', this.handleReadable.bind(this));
+                socket.on('close', this.handleClose.bind(this));
+                socket.on('error', err => this.onerror(err));
+                resolve();
+                this.onopen();
+            });
+            req.on('close', () => {
+                req.removeAllListeners();
+            });
+            req.on('error', e => {
+                if (retries < 5) {
+                    setTimeout(() => {
+                        resolve(this.connect(retries + 1));
+                    }, 500);
+                }
+                else {
+                    rej(e);
+                }
+            });
+            req.end();
         });
-        req.end();
     }
     handleReadable() {
         // Keep reading until no data, this is useful when two payloads merges.
@@ -66,9 +79,11 @@ class SeyfertWebSocket {
                 // Equivalent to readUint32BE
                 length = this.readBytes(2, slice - 2);
             }
+            const payloadLength = slice + length;
             // Read the frame, ignore data next to it, leave it to next `while` cycle
-            const frame = this.socket.read(slice + length);
-            if (!frame)
+            const frame = this.socket.read(payloadLength);
+            // unfinished object when socket closes while reading the data
+            if (!frame || frame.length !== payloadLength)
                 return;
             // Get fin (0 | 1)
             const fin = frame[0] >> 7;
@@ -131,7 +146,7 @@ class SeyfertWebSocket {
                 break;
         }
     }
-    handleClose() {
+    async handleClose() {
         this.socket?.removeAllListeners();
         this.socket?.destroy();
         this.socket = undefined;
@@ -146,6 +161,8 @@ class SeyfertWebSocket {
         this._write(Buffer.from(data), 1);
     }
     _write(buffer, opcode) {
+        if (!this.socket?.writable)
+            return;
         const length = buffer.length;
         let frame;
         // Kinda same logic as above, but client-side
@@ -169,12 +186,24 @@ class SeyfertWebSocket {
         frame.set(buffer, frame.length - length);
         this.socket?.write(frame);
     }
-    onping(_data) { }
-    onpong(_data) { }
-    onopen() { }
-    onmessage(_payload) { }
-    onclose(_close) { }
-    onerror(_err) { }
+    onping(_data) {
+        //
+    }
+    onpong(_data) {
+        //
+    }
+    onopen() {
+        //
+    }
+    onmessage(_payload) {
+        //
+    }
+    onclose(_close) {
+        //
+    }
+    onerror(_err) {
+        //
+    }
     close(code, reason) {
         this.__closeCalled = true;
         // alloc payload length
@@ -186,6 +215,7 @@ class SeyfertWebSocket {
         // message, close opcode
         this._write(buffer, 0x8);
         this.socket?.end();
+        this.onclose({ code, reason });
     }
     pong(data) {
         //send pong opcode (10)

package/lib/websocket/discord/worker.d.ts

@@ -9,13 +9,16 @@ export interface WorkerShardInfo {
 export type WorkerInfo = {
     shards: WorkerShardInfo[];
 };
-type CreateWorkerMessage<T extends string, D extends object = {}> = {
+type CreateWorkerMessage<T extends string, D extends object = object> = {
     type: T;
     workerId: number;
 } & D;
 export type WorkerRequestConnect = CreateWorkerMessage<'CONNECT_QUEUE', {
     shardId: number;
 }>;
+export type WorkerRequestConnectResharding = CreateWorkerMessage<'CONNECT_QUEUE_RESHARDING', {
+    shardId: number;
+}>;
 export type WorkerReceivePayload = CreateWorkerMessage<'RECEIVE_PAYLOAD', {
     shardId: number;
     payload: GatewayDispatchPayload;
@@ -35,7 +38,11 @@ export type WorkerSendInfo = CreateWorkerMessage<'WORKER_INFO', WorkerInfo & {
     nonce: string;
 }>;
 export type WorkerReady = CreateWorkerMessage<'WORKER_READY'>;
+export type WorkerReadyResharding = CreateWorkerMessage<'WORKER_READY_RESHARDING'>;
+export type WorkerShardsConnected = CreateWorkerMessage<'WORKER_SHARDS_CONNECTED'>;
 export type WorkerStart = CreateWorkerMessage<'WORKER_START'>;
+export type WorkerStartResharding = CreateWorkerMessage<'WORKER_START_RESHARDING'>;
+export type WorkerDisconnectedAllShardsResharding = CreateWorkerMessage<'DISCONNECTED_ALL_SHARDS_RESHARDING'>;
 export type WorkerSendApiRequest = CreateWorkerMessage<'WORKER_API_REQUEST', {
     method: HttpMethods;
     url: `/${string}`;
@@ -51,5 +58,5 @@ export type WorkerSendEval = CreateWorkerMessage<'EVAL', {
     nonce: string;
     toWorkerId: number;
 }>;
-export type WorkerMessage = WorkerRequestConnect | WorkerReceivePayload | WorkerSendResultPayload | WorkerSendCacheRequest | WorkerSendShardInfo | WorkerSendInfo | WorkerReady | WorkerSendApiRequest | WorkerSendEvalResponse | WorkerSendEval | WorkerStart;
+export type WorkerMessage = WorkerRequestConnect | WorkerReceivePayload | WorkerSendResultPayload | WorkerSendCacheRequest | WorkerSendShardInfo | WorkerSendInfo | WorkerReady | WorkerShardsConnected | WorkerSendApiRequest | WorkerSendEvalResponse | WorkerSendEval | WorkerStart | WorkerStartResharding | WorkerRequestConnectResharding | WorkerReadyResharding | WorkerDisconnectedAllShardsResharding;
 export {};

package/lib/websocket/discord/workermanager.d.ts

@@ -1,26 +1,38 @@
-import type { GatewayPresenceUpdateData, GatewaySendPayload } from '../../types';
 import { type Worker as ClusterWorker } from 'node:cluster';
 import { ApiHandler, Logger } from '../..';
 import { type Adapter } from '../../cache';
 import { type MakePartial } from '../../common';
+import type { GatewayPresenceUpdateData, GatewaySendPayload } from '../../types';
 import { ConnectQueue } from '../structures/timeout';
 import type { ShardOptions, WorkerData, WorkerManagerOptions } from './shared';
 import type { WorkerInfo, WorkerMessage, WorkerShardInfo } from './worker';
 export declare class WorkerManager extends Map<number, (ClusterWorker | import('node:worker_threads').Worker | {
-    ready: boolean;
+    ready?: boolean;
 }) & {
     ready?: boolean;
+    disconnected?: boolean;
+    resharded?: boolean;
 }> {
+    static prepareSpaces(options: {
+        shardStart: number;
+        shardEnd: number;
+        shardsPerWorker: number;
+    }, logger?: Logger): number[][];
     options: MakePartial<Required<WorkerManagerOptions>, 'adapter'>;
     debugger?: Logger;
     connectQueue: ConnectQueue;
+    workerQueue: (() => void)[];
     cacheAdapter: Adapter;
     promises: Map<string, {
         resolve: (value: any) => void;
         timeout: NodeJS.Timeout;
     }>;
     rest: ApiHandler;
-    constructor(options: MakePartial<WorkerManagerOptions, 'token' | 'intents' | 'info' | 'handlePayload'>);
+    reshardingWorkerQueue: (() => void)[];
+    private _info?;
+    constructor(options: Omit<MakePartial<WorkerManagerOptions, 'token' | 'intents' | 'info' | 'handlePayload'>, 'resharding'> & {
+        resharding?: MakePartial<NonNullable<WorkerManagerOptions['resharding']>, 'getInfo'>;
+    });
     setCache(adapter: Adapter): void;
     setRest(rest: ApiHandler): void;
     get remaining(): number;
@@ -30,20 +42,25 @@ export declare class WorkerManager extends Map<number, (ClusterWorker | import('
     get shardStart(): number;
     get shardEnd(): number;
     get shardsPerWorker(): number;
-    get workers(): number;
-    syncLatency({ shardId, workerId }: {
-        shardId?: number;
+    syncLatency({ shardId, workerId, }: {
+        shardId: number;
         workerId?: number;
-    }): Promise<number | undefined>;
+    } | {
+        shardId?: number;
+        workerId: number;
+    }): Promise<number>;
     calculateShardId(guildId: string): number;
     calculateWorkerId(shardId: number): number;
-    prepareSpaces(): number[][];
-    postMessage(id: number, body: any): void;
-    prepareWorkers(shards: number[][]): Promise<void>;
-    createWorker(workerData: WorkerData): ClusterWorker | import("worker_threads").Worker | {
-        ready: boolean;
-    };
-    spawn(workerId: number, shardId: number): void;
+    postMessage(id: number, body: ManagerMessages): void;
+    prepareWorkers(shards: number[][], resharding?: boolean): Promise<void>;
+    createWorker(workerData: WorkerData): ClusterWorker | import("worker_threads").Worker | ({
+        ready?: boolean;
+    } & {
+        ready?: boolean;
+        disconnected?: boolean;
+        resharded?: boolean;
+    });
+    spawn(workerId: number, shardId: number, resharding?: boolean): void;
     handleWorkerMessage(message: WorkerMessage): Promise<void>;
     private generateNonce;
     private generateSendPromise;
@@ -51,15 +68,24 @@ export declare class WorkerManager extends Map<number, (ClusterWorker | import('
     getShardInfo(shardId: number): Promise<WorkerShardInfo>;
     getWorkerInfo(workerId: number): Promise<WorkerInfo>;
     start(): Promise<void>;
+    startResharding(): Promise<void>;
 }
-type CreateManagerMessage<T extends string, D extends object = {}> = {
+type CreateManagerMessage<T extends string, D extends object = object> = {
     type: T;
 } & D;
 export type ManagerAllowConnect = CreateManagerMessage<'ALLOW_CONNECT', {
     shardId: number;
     presence: GatewayPresenceUpdateData;
 }>;
+export type ManagerAllowConnectResharding = CreateManagerMessage<'ALLOW_CONNECT_RESHARDING', {
+    shardId: number;
+    presence: GatewayPresenceUpdateData;
+}>;
+export type ManagerWorkerAlreadyExistsResharding = CreateManagerMessage<'WORKER_ALREADY_EXISTS_RESHARDING'>;
 export type ManagerSpawnShards = CreateManagerMessage<'SPAWN_SHARDS', Pick<ShardOptions, 'info' | 'properties' | 'compress'>>;
+export type ManagerSpawnShardsResharding = CreateManagerMessage<'SPAWN_SHARDS_RESHARDING', Pick<ShardOptions, 'info' | 'properties' | 'compress'>>;
+export type DisconnectAllShardsResharding = CreateManagerMessage<'DISCONNECT_ALL_SHARDS_RESHARDING'>;
+export type ConnnectAllShardsResharding = CreateManagerMessage<'CONNECT_ALL_SHARDS_RESHARDING'>;
 export type ManagerSendPayload = CreateManagerMessage<'SEND_PAYLOAD', GatewaySendPayload & {
     shardId: number;
     nonce: string;
@@ -90,5 +116,5 @@ export type ManagerSendEvalResponse = CreateManagerMessage<'EVAL_RESPONSE', {
     response: any;
     nonce: string;
 }>;
-export type ManagerMessages = ManagerAllowConnect | ManagerSpawnShards | ManagerSendPayload | ManagerRequestShardInfo | ManagerRequestWorkerInfo | ManagerSendCacheResult | ManagerSendBotReady | ManagerSendApiResponse | ManagerSendEvalResponse | ManagerExecuteEval;
+export type ManagerMessages = ManagerAllowConnect | ManagerSpawnShards | ManagerSendPayload | ManagerRequestShardInfo | ManagerRequestWorkerInfo | ManagerSendCacheResult | ManagerSendBotReady | ManagerSendApiResponse | ManagerSendEvalResponse | ManagerExecuteEval | ManagerWorkerAlreadyExistsResharding | ManagerSpawnShardsResharding | ManagerAllowConnectResharding | DisconnectAllShardsResharding | ConnnectAllShardsResharding;
 export {};