seyfert 2.0.0 → 2.1.1-dev-11310514874.0

package/lib/websocket/discord/workermanager.js

@@ -14,12 +14,27 @@ const constants_1 = require("../constants");
 const structures_1 = require("../structures");
 const timeout_1 = require("../structures/timeout");
 class WorkerManager extends Map {
+    static prepareSpaces(options, logger) {
+        logger?.info('Preparing buckets', options);
+        const chunks = structures_1.DynamicBucket.chunk(new Array(options.shardEnd - options.shardStart), options.shardsPerWorker);
+        chunks.forEach((shards, index) => {
+            for (let i = 0; i < shards.length; i++) {
+                const id = i + (index > 0 ? index * options.shardsPerWorker : 0) + options.shardStart;
+                chunks[index][i] = id;
+            }
+        });
+        logger?.info(`${chunks.length} buckets created`);
+        return chunks;
+    }
     options;
     debugger;
     connectQueue;
+    workerQueue = [];
     cacheAdapter;
     promises = new Map();
     rest;
+    reshardingWorkerQueue = [];
+    _info;
     constructor(options) {
         super();
         this.options = options;
@@ -52,12 +67,9 @@ class WorkerManager extends Map {
     get shardsPerWorker() {
         return this.options.shardsPerWorker;
     }
-    get workers() {
-        return this.options.workers;
-    }
-    async syncLatency({ shardId, workerId }) {
+    async syncLatency({ shardId, workerId, }) {
         if (typeof shardId !== 'number' && typeof workerId !== 'number') {
-            return;
+            throw new Error('Undefined workerId and shardId');
         }
         const id = workerId ?? this.calculateWorkerId(shardId);
         if (!this.has(id)) {
@@ -71,27 +83,15 @@ class WorkerManager extends Map {
     }
     calculateWorkerId(shardId) {
         const workerId = Math.floor((shardId - this.shardStart) / this.shardsPerWorker);
-        if (workerId >= this.workers) {
+        if (workerId >= this.totalWorkers) {
             throw new Error('Invalid shardId');
         }
         return workerId;
     }
-    prepareSpaces() {
-        this.debugger?.info('Preparing buckets');
-        const chunks = structures_1.DynamicBucket.chunk(new Array(this.shardEnd - this.shardStart), this.options.shardsPerWorker);
-        chunks.forEach((shards, index) => {
-            for (let i = 0; i < shards.length; i++) {
-                const id = i + (index > 0 ? index * this.options.shardsPerWorker : 0) + this.shardStart;
-                chunks[index][i] = id;
-            }
-        });
-        this.debugger?.info(`${chunks.length} buckets created`);
-        return chunks;
-    }
     postMessage(id, body) {
         const worker = this.get(id);
         if (!worker)
-            return this.debugger?.error(`Worker ${id} doesnt exists.`);
+            return this.debugger?.error(`Worker ${id} does not exists.`);
         switch (this.options.mode) {
             case 'clusters':
                 worker.send(body);
@@ -104,35 +104,49 @@ class WorkerManager extends Map {
                 break;
         }
     }
-    async prepareWorkers(shards) {
+    async prepareWorkers(shards, resharding = false) {
         const worker_threads = (0, common_1.lazyLoadPackage)('node:worker_threads');
         if (!worker_threads)
             throw new Error('Cannot prepare workers without worker_threads.');
         for (let i = 0; i < shards.length; i++) {
-            let worker = this.get(i);
-            if (!worker) {
-                worker = this.createWorker({
-                    path: this.options.path,
-                    debug: this.options.debug,
-                    token: this.options.token,
-                    shards: shards[i],
-                    intents: this.options.intents,
-                    workerId: i,
-                    workerProxy: this.options.workerProxy,
-                    totalShards: this.totalShards,
-                    mode: this.options.mode,
+            const workerExists = this.has(i);
+            if (resharding || !workerExists) {
+                this[resharding ? 'reshardingWorkerQueue' : 'workerQueue'].push(() => {
+                    const worker = this.createWorker({
+                        path: this.options.path,
+                        debug: this.options.debug,
+                        token: this.options.token,
+                        shards: shards[i],
+                        intents: this.options.intents,
+                        workerId: i,
+                        workerProxy: this.options.workerProxy,
+                        totalShards: resharding ? this._info.shards : this.totalShards,
+                        mode: this.options.mode,
+                        resharding,
+                    });
+                    this.set(i, worker);
                 });
-                this.set(i, worker);
             }
         }
     }
     createWorker(workerData) {
+        if (this.has(workerData.workerId)) {
+            if (workerData.resharding) {
+                this.postMessage(workerData.workerId, {
+                    type: 'WORKER_ALREADY_EXISTS_RESHARDING',
+                });
+            }
+            const worker = this.get(workerData.workerId);
+            return worker;
+        }
         const worker_threads = (0, common_1.lazyLoadPackage)('node:worker_threads');
         if (!worker_threads)
             throw new Error('Cannot create worker without worker_threads.');
         const env = {
             SEYFERT_SPAWNING: 'true',
         };
+        if (workerData.resharding)
+            env.SEYFERT_WORKER_RESHARDING = 'true';
         for (const i in workerData) {
             env[`SEYFERT_WORKER_${i.toUpperCase()}`] = workerData[i];
         }
@@ -152,22 +166,23 @@ class WorkerManager extends Map {
                 worker.on('message', data => this.handleWorkerMessage(data));
                 return worker;
             }
-            case 'custom':
+            case 'custom': {
                 this.options.adapter.spawn(workerData, env);
                 return {
                     ready: false,
                 };
+            }
         }
     }
-    spawn(workerId, shardId) {
+    spawn(workerId, shardId, resharding = false) {
         this.connectQueue.push(() => {
             const worker = this.has(workerId);
             if (!worker) {
-                this.debugger?.fatal("Trying spawn with worker doesn't exist");
+                this.debugger?.fatal(`Trying ${resharding ? 'reshard' : 'spawn'} with worker that doesn't exist`);
                 return;
             }
             this.postMessage(workerId, {
-                type: 'ALLOW_CONNECT',
+                type: resharding ? 'ALLOW_CONNECT_RESHARDING' : 'ALLOW_CONNECT',
                 shardId,
                 presence: this.options.presence?.(shardId, workerId),
             });
@@ -175,6 +190,66 @@ class WorkerManager extends Map {
     }
     async handleWorkerMessage(message) {
         switch (message.type) {
+            case 'WORKER_READY_RESHARDING':
+                {
+                    this.get(message.workerId).resharded = true;
+                    if (!this.reshardingWorkerQueue.length && [...this.values()].every(w => w.resharded)) {
+                        for (const [id] of this.entries()) {
+                            this.postMessage(id, {
+                                type: 'DISCONNECT_ALL_SHARDS_RESHARDING',
+                            });
+                        }
+                        this.forEach(w => {
+                            delete w.resharded;
+                        });
+                    }
+                    else {
+                        const nextWorker = this.reshardingWorkerQueue.shift();
+                        if (nextWorker) {
+                            this.debugger?.info('Spawning next worker to reshard');
+                            nextWorker();
+                        }
+                        else {
+                            this.debugger?.info('No more workers to reshard left');
+                        }
+                    }
+                }
+                break;
+            case 'DISCONNECTED_ALL_SHARDS_RESHARDING':
+                {
+                    this.get(message.workerId).disconnected = true;
+                    if ([...this.values()].every(w => w.disconnected)) {
+                        this.options.totalShards = this._info.shards;
+                        this.options.shardEnd = this.options.totalShards = this._info.shards;
+                        this.options.workers = this.size;
+                        delete this._info;
+                        for (const [id] of this.entries()) {
+                            this.postMessage(id, {
+                                type: 'CONNECT_ALL_SHARDS_RESHARDING',
+                            });
+                        }
+                        this.forEach(w => {
+                            delete w.disconnected;
+                        });
+                    }
+                }
+                break;
+            case 'WORKER_START_RESHARDING':
+                {
+                    this.postMessage(message.workerId, {
+                        type: 'SPAWN_SHARDS_RESHARDING',
+                        compress: this.options.compress ?? false,
+                        info: {
+                            ...this.options.info,
+                            shards: this._info.shards,
+                        },
+                        properties: {
+                            ...constants_1.properties,
+                            ...this.options.properties,
+                        },
+                    });
+                }
+                break;
             case 'WORKER_START':
                 {
                     this.postMessage(message.workerId, {
@@ -191,6 +266,9 @@ class WorkerManager extends Map {
                     });
                 }
                 break;
+            case 'CONNECT_QUEUE_RESHARDING':
+                this.spawn(message.workerId, message.shardId, true);
+                break;
             case 'CONNECT_QUEUE':
                 this.spawn(message.workerId, message.shardId);
                 break;
@@ -260,6 +338,18 @@ class WorkerManager extends Map {
                     }
                 }
                 break;
+            case 'WORKER_SHARDS_CONNECTED':
+                {
+                    const nextWorker = this.workerQueue.shift();
+                    if (nextWorker) {
+                        this.debugger?.info('Spawning next worker');
+                        nextWorker();
+                    }
+                    else {
+                        this.debugger?.info('No more workers to spawn left');
+                    }
+                }
+                break;
             case 'WORKER_API_REQUEST':
                 {
                     const response = await this.rest.request(message.method, message.url, message.requestOptions);
@@ -300,12 +390,11 @@ class WorkerManager extends Map {
                 break;
         }
     }
-    generateNonce(large = true) {
+    generateNonce() {
         const uuid = (0, node_crypto_1.randomUUID)();
-        const nonce = large ? uuid : uuid.split('-')[0];
-        if (this.promises.has(nonce))
-            return this.generateNonce(large);
-        return nonce;
+        if (this.promises.has(uuid))
+            return this.generateNonce();
+        return uuid;
     }
     generateSendPromise(nonce, message = 'Timeout') {
         return new Promise((res, rej) => {
@@ -337,7 +426,7 @@ class WorkerManager extends Map {
         if (!worker) {
             throw new Error(`Worker #${workerId} doesnt exist`);
         }
-        const nonce = this.generateNonce(false);
+        const nonce = this.generateNonce();
         this.postMessage(workerId, { shardId, nonce, type: 'SHARD_INFO' });
         return this.generateSendPromise(nonce, 'Get shard info timeout');
     }
@@ -361,10 +450,11 @@ class WorkerManager extends Map {
             domain: 'https://discord.com',
             debug: this.options.debug,
         });
-        this.options.info ??= await new __1.Router(this.rest).createProxy().gateway.bot.get();
+        this.options.info ??= await this.rest.proxy.gateway.bot.get();
         this.options.shardEnd ??= this.options.totalShards ?? this.options.info.shards;
         this.options.totalShards ??= this.options.shardEnd;
         this.options = (0, common_1.MergeOptions)(constants_1.WorkerManagerDefaults, this.options);
+        this.options.resharding.getInfo ??= () => this.rest.proxy.gateway.bot.get();
         this.options.workers ??= Math.ceil(this.options.totalShards / this.options.shardsPerWorker);
         this.connectQueue = new timeout_1.ConnectQueue(5.5e3, this.concurrency);
         if (this.options.debug) {
@@ -372,11 +462,45 @@ class WorkerManager extends Map {
                 name: '[WorkerManager]',
             });
         }
-        if (this.totalShards / this.shardsPerWorker > this.workers) {
+        if (this.totalShards / this.shardsPerWorker > this.totalWorkers) {
             throw new Error(`Cannot create enough shards in the specified workers, minimum: ${Math.ceil(this.totalShards / this.shardsPerWorker)}`);
         }
-        const spaces = this.prepareSpaces();
+        const spaces = WorkerManager.prepareSpaces({
+            shardStart: this.shardStart,
+            shardEnd: this.shardEnd,
+            shardsPerWorker: this.shardsPerWorker,
+        }, this.debugger);
         await this.prepareWorkers(spaces);
+        // Start workers queue
+        this.workerQueue.shift()();
+        await this.startResharding();
+    }
+    async startResharding() {
+        if (this.options.resharding.interval <= 0)
+            return;
+        if (this.shardStart !== 0 || this.shardEnd !== this.totalShards)
+            return this.debugger?.debug('Cannot start resharder');
+        setInterval(async () => {
+            this.debugger?.debug('Checking if reshard is needed');
+            const info = await this.options.resharding.getInfo();
+            if (info.shards <= this.totalShards)
+                return this.debugger?.debug('Resharding not needed');
+            //https://github.com/discordeno/discordeno/blob/6a5f446c0651b9fad9f1550ff1857fe7a026426b/packages/gateway/src/manager.ts#L106C8-L106C94
+            const percentage = (info.shards / ((this.totalShards * 2500) / 1000)) * 100;
+            if (percentage < this.options.resharding.percentage)
+                return this.debugger?.debug(`Percentage is not enough to reshard ${percentage}/${this.options.resharding.percentage}`);
+            this.debugger?.info('Starting resharding process');
+            this._info = info;
+            this.connectQueue.concurrency = info.session_start_limit.max_concurrency;
+            this.options.info.session_start_limit.max_concurrency = info.session_start_limit.max_concurrency;
+            const spaces = WorkerManager.prepareSpaces({
+                shardsPerWorker: this.shardsPerWorker,
+                shardEnd: info.shards,
+                shardStart: 0,
+            }, this.debugger);
+            await this.prepareWorkers(spaces, true);
+            return this.reshardingWorkerQueue.shift()();
+        }, this.options.resharding.interval);
     }
 }
 exports.WorkerManager = WorkerManager;

package/lib/websocket/structures/timeout.d.ts

@@ -9,11 +9,10 @@ export declare class ConnectTimeout {
 export declare class ConnectQueue {
     intervalTime: number;
     concurrency: number;
-    readonly queue: {
-        cb: (() => any) | undefined;
-    }[];
+    private queue;
+    private remaining;
     protected interval?: NodeJS.Timeout;
     constructor(intervalTime?: number, concurrency?: number);
-    push(callback: () => any): Promise<void>;
-    shift(): Promise<any>;
+    push(callback: () => unknown): unknown;
+    startInterval(): void;
 }

package/lib/websocket/structures/timeout.js

@@ -32,41 +32,40 @@ class ConnectQueue {
     intervalTime;
     concurrency;
     queue = [];
+    remaining = 0;
     interval = undefined;
     constructor(intervalTime = 5000, concurrency = 1) {
         this.intervalTime = intervalTime;
         this.concurrency = concurrency;
+        this.remaining = concurrency;
     }
-    async push(callback) {
-        this.queue.push({ cb: callback });
-        if (this.queue.length === this.concurrency) {
-            for (let i = 0; i < this.concurrency; i++) {
-                await this.queue[i].cb?.();
-                this.queue[i].cb = undefined;
-            }
-            this.interval = setInterval(() => {
-                for (let i = 0; i < this.concurrency; i++) {
-                    this.shift();
-                }
-            }, this.intervalTime);
+    push(callback) {
+        if (this.remaining === 0)
+            return this.queue.push(callback);
+        this.remaining--;
+        if (!this.interval) {
+            this.startInterval();
         }
+        if (this.queue.length < this.concurrency) {
+            return callback();
+        }
+        return this.queue.push(callback);
     }
-    async shift() {
-        const shift = this.queue.shift();
-        if (!shift) {
+    startInterval() {
+        this.interval = setInterval(() => {
+            let cb;
+            while (this.queue.length && !(cb = this.queue.shift())) {
+                //
+            }
+            if (cb)
+                return cb?.();
+            if (this.remaining < this.concurrency)
+                return this.remaining++;
             if (!this.queue.length) {
                 clearInterval(this.interval);
                 this.interval = undefined;
             }
-            return;
-        }
-        if (!shift.cb)
-            return this.shift();
-        await shift.cb?.();
-        if (!this.queue.length) {
-            clearInterval(this.interval);
-            this.interval = undefined;
-        }
+        }, this.intervalTime / this.concurrency);
     }
 }
 exports.ConnectQueue = ConnectQueue;

package/package.json

@@ -1,65 +1,72 @@
-{
-	"name": "seyfert",
-	"version": "2.0.0",
-	"description": "The most advanced framework for discord bots",
-	"main": "./lib/index.js",
-	"module": "./lib/index.js",
-	"types": "./lib/index.d.ts",
-	"files": [
-		"lib/**"
-	],
-	"scripts": {
-		"build": "tsc --outDir ./lib",
-		"prepublishOnly": "npm run build",
-		"prepare": "npm run build && husky",
-		"lint": "biome lint --write ./src",
-		"format": "biome format --write ./src",
-		"check-h": "biome check --write ./src",
-		"check": "biome check --write --no-errors-on-unmatched ./src"
-	},
-	"author": "MARCROCK22",
-	"license": "Apache-2.0",
-	"devDependencies": {
-		"@biomejs/biome": "1.8.3",
-		"@commitlint/cli": "^19.4.0",
-		"@commitlint/config-conventional": "^19.2.2",
-		"@types/node": "^22.5.0",
-		"husky": "^9.1.5",
-		"lint-staged": "^15.2.9",
-		"typescript": "^5.5.4"
-	},
-	"homepage": "https://seyfert.dev",
-	"repository": {
-		"type": "git",
-		"url": "git+https://github.com/tiramisulabs/seyfert.git"
-	},
-	"bugs": {
-		"url": "https://github.com/tiramisulabs/seyfert"
-	},
-	"keywords": [
-		"api",
-		"discord",
-		"bots",
-		"typescript",
-		"botdev"
-	],
-	"publishConfig": {
-		"access": "public"
-	},
-	"maintainers": [
-		{
-			"name": "socram03",
-			"url": "https://github.com/socram03"
-		}
-	],
-	"contributors": [
-		{
-			"name": "Free 公園",
-			"url": "https://github.com/FreeAoi"
-		},
-		{
-			"name": "David",
-			"url": "https://github.com/Drylozu"
-		}
-	]
-}
+{
+	"name": "seyfert",
+	"version": "2.1.1-dev-11310514874.0",
+	"description": "The most advanced framework for discord bots",
+	"main": "./lib/index.js",
+	"module": "./lib/index.js",
+	"types": "./lib/index.d.ts",
+	"files": [
+		"lib/**"
+	],
+	"scripts": {
+		"build": "tsc --outDir ./lib",
+		"prepublishOnly": "npm run build",
+		"prepare": "npm run build && husky",
+		"lint": "biome lint --write ./src",
+		"format": "biome format --write ./src",
+		"check-h": "biome check --write ./src",
+		"check": "biome check --write --no-errors-on-unmatched ./src",
+		"test": "vitest run --config ./tests/vitest.config.mts ./tests/"
+	},
+	"author": "MARCROCK22",
+	"license": "MIT",
+	"devDependencies": {
+		"@biomejs/biome": "1.9.3",
+		"@commitlint/cli": "^19.5.0",
+		"@commitlint/config-conventional": "^19.5.0",
+		"@types/node": "^22.7.4",
+		"husky": "^9.1.6",
+		"lint-staged": "^15.2.10",
+		"typescript": "^5.6.2",
+		"vitest": "^2.1.2"
+	},
+	"homepage": "https://seyfert.dev",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/tiramisulabs/seyfert.git"
+	},
+	"bugs": {
+		"url": "https://github.com/tiramisulabs/seyfert"
+	},
+	"keywords": [
+		"api",
+		"discord",
+		"bots",
+		"typescript",
+		"botdev"
+	],
+	"publishConfig": {
+		"access": "public"
+	},
+	"maintainers": [
+		{
+			"name": "socram03",
+			"url": "https://github.com/socram03"
+		}
+	],
+	"contributors": [
+		{
+			"name": "Free 公園",
+			"url": "https://github.com/FreeAoi"
+		},
+		{
+			"name": "David",
+			"url": "https://github.com/Drylozu"
+		}
+	],
+	"lint-staged": {
+		"*.ts": [
+			"biome check --write"
+		]
+	}
+}

package/lib/cache/resources/threads.d.ts

@@ -1,14 +0,0 @@
-import type { APIThreadChannel } from '../../types';
-import type { ReturnCache } from '../..';
-import { GuildRelatedResource } from './default/guild-related';
-import { type ThreadChannelStructure } from '../../client/transformers';
-export declare class Threads extends GuildRelatedResource<any, APIThreadChannel> {
-    namespace: string;
-    filter(data: APIThreadChannel, id: string, guild_id?: string): boolean;
-    get(id: string): ReturnCache<ThreadChannelStructure | undefined>;
-    raw(id: string): ReturnCache<APIThreadChannel | undefined>;
-    bulk(ids: string[]): ReturnCache<ThreadChannelStructure[]>;
-    bulkRaw(ids: string[]): ReturnCache<APIThreadChannel[]>;
-    values(guild: string): ReturnCache<ThreadChannelStructure[]>;
-    valuesRaw(guild: string): ReturnCache<APIThreadChannel[]>;
-}

package/lib/cache/resources/threads.js

@@ -1,32 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Threads = void 0;
-const common_1 = require("../../common");
-const guild_related_1 = require("./default/guild-related");
-const transformers_1 = require("../../client/transformers");
-class Threads extends guild_related_1.GuildRelatedResource {
-    namespace = 'thread';
-    //@ts-expect-error
-    filter(data, id, guild_id) {
-        return true;
-    }
-    get(id) {
-        return (0, common_1.fakePromise)(super.get(id)).then(rawThread => rawThread ? transformers_1.Transformers.ThreadChannel(this.client, rawThread) : undefined);
-    }
-    raw(id) {
-        return super.get(id);
-    }
-    bulk(ids) {
-        return (0, common_1.fakePromise)(super.bulk(ids)).then(threads => threads.map(rawThread => transformers_1.Transformers.ThreadChannel(this.client, rawThread)));
-    }
-    bulkRaw(ids) {
-        return super.bulk(ids);
-    }
-    values(guild) {
-        return (0, common_1.fakePromise)(super.values(guild)).then(threads => threads.map(rawThread => transformers_1.Transformers.ThreadChannel(this.client, rawThread)));
-    }
-    valuesRaw(guild) {
-        return super.values(guild);
-    }
-}
-exports.Threads = Threads;