seyfert 0.1.0

package/lib/websocket/discord/worker.d.ts

@@ -0,0 +1,43 @@
+import type { GatewayDispatchPayload } from '../../common';
+export interface WorkerShardInfo {
+    open: boolean;
+    shardId: number;
+    latency: number;
+    resumable: boolean;
+}
+export type WorkerInfo = {
+    shards: WorkerShardInfo[];
+    workerId: number;
+};
+type CreateWorkerMessage<T extends string, D extends object = {}> = {
+    type: T;
+} & D;
+export type WorkerRequestConnect = CreateWorkerMessage<'CONNECT_QUEUE', {
+    shardId: number;
+    workerId: number;
+}>;
+export type WorkerReceivePayload = CreateWorkerMessage<'RECEIVE_PAYLOAD', {
+    shardId: number;
+    workerId: number;
+    payload: GatewayDispatchPayload;
+}>;
+export type WorkerSendResultPayload = CreateWorkerMessage<'RESULT_PAYLOAD', {
+    nonce: string;
+}>;
+export type WorkerSendCacheRequest = CreateWorkerMessage<'CACHE_REQUEST', {
+    nonce: string;
+    method: 'scan' | 'get' | 'set' | 'patch' | 'values' | 'keys' | 'count' | 'remove' | 'contains' | 'getToRelationship' | 'bulkAddToRelationShip' | 'addToRelationship' | 'removeRelationship' | 'removeToRelationship';
+    args: any[];
+    workerId: number;
+}>;
+export type WorkerSendShardInfo = CreateWorkerMessage<'SHARD_INFO', WorkerShardInfo & {
+    nonce: string;
+}>;
+export type WorkerSendInfo = CreateWorkerMessage<'WORKER_INFO', WorkerInfo & {
+    nonce: string;
+}>;
+export type WorkerReady = CreateWorkerMessage<'WORKER_READY', {
+    workerId: number;
+}>;
+export type WorkerMessage = WorkerRequestConnect | WorkerReceivePayload | WorkerSendResultPayload | WorkerSendCacheRequest | WorkerSendShardInfo | WorkerSendInfo | WorkerReady;
+export {};

package/lib/websocket/discord/worker.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/websocket/discord/workermanager.d.ts

@@ -0,0 +1,69 @@
+/// <reference types="node" />
+import { Worker } from 'node:worker_threads';
+import { type Adapter } from '../../cache';
+import { Logger, type GatewayPresenceUpdateData, type GatewaySendPayload } from '../../common';
+import { ConnectQueue } from '../structures/timeout';
+import { MemberUpdateHandler } from './events/memberUpdate';
+import { PresenceUpdateHandler } from './events/presenceUpdate';
+import type { ShardOptions, WorkerData, WorkerManagerOptions } from './shared';
+import type { WorkerInfo, WorkerMessage, WorkerShardInfo } from './worker';
+export declare class WorkerManager extends Map<number, Worker> {
+    options: Required<WorkerManagerOptions>;
+    debugger?: Logger;
+    connectQueue: ConnectQueue;
+    cacheAdapter: Adapter;
+    promises: Map<string, (value: any) => void>;
+    memberUpdateHandler: MemberUpdateHandler;
+    presenceUpdateHandler: PresenceUpdateHandler;
+    constructor(options: WorkerManagerOptions);
+    setCache(adapter: Adapter): void;
+    get remaining(): number;
+    get concurrency(): number;
+    get totalWorkers(): number;
+    get totalShards(): number;
+    get shardsPerWorker(): number;
+    get workers(): number;
+    syncLatency({ shardId, workerId }: {
+        shardId?: number;
+        workerId?: number;
+    }): Promise<number | undefined>;
+    calculateShardId(guildId: string): number;
+    calculateWorkerId(shardId: number): number;
+    prepareSpaces(): number[][];
+    prepareWorkers(shards: number[][]): Promise<void>;
+    createWorker(workerData: WorkerData): Worker;
+    spawn(workerId: number, shardId: number): void;
+    handleWorkerMessage(message: WorkerMessage): Promise<void>;
+    private generateNonce;
+    private generateSendPromise;
+    send(data: GatewaySendPayload, shardId: number): Promise<true>;
+    getShardInfo(shardId: number): Promise<WorkerShardInfo>;
+    getWorkerInfo(workerId: number): Promise<WorkerInfo>;
+    start(): Promise<void>;
+}
+type CreateManagerMessage<T extends string, D extends object = {}> = {
+    type: T;
+} & D;
+export type ManagerAllowConnect = CreateManagerMessage<'ALLOW_CONNECT', {
+    shardId: number;
+    presence: GatewayPresenceUpdateData;
+}>;
+export type ManagerSpawnShards = CreateManagerMessage<'SPAWN_SHARDS', Pick<ShardOptions, 'info' | 'properties' | 'compress'>>;
+export type ManagerSendPayload = CreateManagerMessage<'SEND_PAYLOAD', GatewaySendPayload & {
+    shardId: number;
+    nonce: string;
+}>;
+export type ManagerRequestShardInfo = CreateManagerMessage<'SHARD_INFO', {
+    nonce: string;
+    shardId: number;
+}>;
+export type ManagerRequestWorkerInfo = CreateManagerMessage<'WORKER_INFO', {
+    nonce: string;
+}>;
+export type ManagerSendCacheResult = CreateManagerMessage<'CACHE_RESULT', {
+    nonce: string;
+    result: any;
+}>;
+export type ManagerSendBotReady = CreateManagerMessage<'BOT_READY'>;
+export type ManagerMessages = ManagerAllowConnect | ManagerSpawnShards | ManagerSendPayload | ManagerRequestShardInfo | ManagerRequestWorkerInfo | ManagerSendCacheResult | ManagerSendBotReady;
+export {};

package/lib/websocket/discord/workermanager.js

@@ -0,0 +1,273 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerManager = void 0;
+const node_crypto_1 = require("node:crypto");
+const node_worker_threads_1 = require("node:worker_threads");
+const cache_1 = require("../../cache");
+const common_1 = require("../../common");
+const constants_1 = require("../constants");
+const structures_1 = require("../structures");
+const timeout_1 = require("../structures/timeout");
+const memberUpdate_1 = require("./events/memberUpdate");
+const presenceUpdate_1 = require("./events/presenceUpdate");
+class WorkerManager extends Map {
+    options;
+    debugger;
+    connectQueue;
+    cacheAdapter;
+    promises = new Map();
+    memberUpdateHandler = new memberUpdate_1.MemberUpdateHandler();
+    presenceUpdateHandler = new presenceUpdate_1.PresenceUpdateHandler();
+    constructor(options) {
+        super();
+        options.totalShards ??= options.info.shards;
+        this.options = (0, common_1.MergeOptions)(constants_1.WorkerManagerDefaults, options);
+        this.options.workers ??= Math.ceil(this.options.totalShards / this.options.shardsPerWorker);
+        this.options.info.shards = options.totalShards;
+        options.shardEnd ??= options.totalShards;
+        options.shardStart ??= 0;
+        this.connectQueue = new timeout_1.ConnectQueue(5.5e3, this.concurrency);
+        if (this.options.debug) {
+            this.debugger = new common_1.Logger({
+                name: '[WorkerManager]',
+            });
+        }
+        if (this.totalShards / this.shardsPerWorker > this.workers) {
+            throw new Error(`Cannot create enough shards in the specified workers, minimum: ${Math.ceil(this.totalShards / this.shardsPerWorker)}`);
+        }
+        this.cacheAdapter = new cache_1.MemoryAdapter();
+    }
+    setCache(adapter) {
+        this.cacheAdapter = adapter;
+    }
+    get remaining() {
+        return this.options.info.session_start_limit.remaining;
+    }
+    get concurrency() {
+        return this.options.info.session_start_limit.max_concurrency;
+    }
+    get totalWorkers() {
+        return this.options.workers;
+    }
+    get totalShards() {
+        return this.options.totalShards;
+    }
+    get shardsPerWorker() {
+        return this.options.shardsPerWorker;
+    }
+    get workers() {
+        return this.options.workers;
+    }
+    async syncLatency({ shardId, workerId }) {
+        if (typeof shardId !== 'number' && typeof workerId !== 'number') {
+            return;
+        }
+        const id = workerId ?? this.calculateWorkerId(shardId);
+        if (!this.has(id)) {
+            throw new Error(`Worker #${workerId} doesnt exist`);
+        }
+        const data = await this.getWorkerInfo(id);
+        return data.shards.reduce((acc, prv) => acc + prv.latency, 0) / data.shards.length;
+    }
+    calculateShardId(guildId) {
+        return Number((BigInt(guildId) >> 22n) % BigInt(this.options.info.shards ?? 1));
+    }
+    calculateWorkerId(shardId) {
+        const workerId = Math.floor(shardId / this.shardsPerWorker);
+        if (workerId >= this.workers) {
+            throw new Error('Invalid shardId');
+        }
+        return workerId;
+    }
+    prepareSpaces() {
+        this.debugger?.info('Preparing buckets');
+        const chunks = structures_1.SequentialBucket.chunk(new Array(this.options.shardStart !== undefined && this.options.shardEnd !== undefined
+            ? this.options.shardEnd - this.options.shardStart
+            : this.options.totalShards), this.options.shardsPerWorker);
+        chunks.forEach((shards, index) => {
+            for (let i = 0; i < shards.length; i++) {
+                const id = i + (index > 0 ? index * this.options.shardsPerWorker : 0) + (this.options.shardStart ?? 0);
+                chunks[index][i] = id;
+            }
+        });
+        this.debugger?.info(`${chunks.length} buckets created`);
+        return chunks;
+    }
+    async prepareWorkers(shards) {
+        for (let i = 0; i < shards.length; i++) {
+            let worker = this.get(i);
+            if (!worker) {
+                worker = this.createWorker({
+                    path: this.options.path,
+                    debug: this.options.debug,
+                    token: this.options.token,
+                    shards: shards[i],
+                    intents: this.options.intents,
+                    workerId: i,
+                });
+                this.set(i, worker);
+            }
+            worker.postMessage({
+                type: 'SPAWN_SHARDS',
+                compress: this.options.compress ?? false,
+                info: this.options.info,
+                properties: this.options.properties,
+            });
+        }
+    }
+    createWorker(workerData) {
+        const worker = new node_worker_threads_1.Worker(workerData.path, { workerData });
+        worker.on('message', data => this.handleWorkerMessage(data));
+        return worker;
+    }
+    spawn(workerId, shardId) {
+        this.connectQueue.push(() => {
+            const worker = this.get(workerId);
+            if (!worker) {
+                this.debugger?.fatal("Trying spawn with worker doesn't exist");
+                return;
+            }
+            worker.postMessage({
+                type: 'ALLOW_CONNECT',
+                shardId,
+                presence: this.options.presence?.(shardId, workerId),
+            });
+        });
+    }
+    async handleWorkerMessage(message) {
+        switch (message.type) {
+            case 'CONNECT_QUEUE':
+                this.spawn(message.workerId, message.shardId);
+                break;
+            case 'CACHE_REQUEST':
+                {
+                    const worker = this.get(message.workerId);
+                    if (!worker) {
+                        throw new Error('Invalid request from unavailable worker');
+                    }
+                    // @ts-expect-error
+                    const result = await this.cacheAdapter[message.method](...message.args);
+                    worker.postMessage({
+                        type: 'CACHE_RESULT',
+                        nonce: message.nonce,
+                        result,
+                    });
+                }
+                break;
+            case 'RECEIVE_PAYLOAD':
+                {
+                    switch (message.payload.t) {
+                        case 'GUILD_MEMBER_UPDATE':
+                            if (!this.memberUpdateHandler.check(message.payload.d)) {
+                                return;
+                            }
+                            break;
+                        case 'PRESENCE_UPDATE':
+                            if (!this.presenceUpdateHandler.check(message.payload.d)) {
+                                return;
+                            }
+                            break;
+                    }
+                    this.options.handlePayload(message.shardId, message.workerId, message.payload);
+                }
+                break;
+            case 'RESULT_PAYLOAD':
+                {
+                    const resolve = this.promises.get(message.nonce);
+                    if (!resolve) {
+                        return;
+                    }
+                    this.promises.delete(message.nonce);
+                    resolve(true);
+                }
+                break;
+            case 'SHARD_INFO':
+                {
+                    const { nonce, type, ...data } = message;
+                    const resolve = this.promises.get(nonce);
+                    if (!resolve) {
+                        return;
+                    }
+                    this.promises.delete(nonce);
+                    resolve(data);
+                }
+                break;
+            case 'WORKER_INFO':
+                {
+                    const { nonce, type, ...data } = message;
+                    const resolve = this.promises.get(nonce);
+                    if (!resolve) {
+                        return;
+                    }
+                    this.promises.delete(nonce);
+                    resolve(data);
+                }
+                break;
+            case 'WORKER_READY':
+                {
+                    if (message.workerId === [...this.keys()].at(-1)) {
+                        this.get(this.keys().next().value)?.postMessage({
+                            type: 'BOT_READY',
+                        });
+                    }
+                }
+                break;
+        }
+    }
+    generateNonce(large = true) {
+        const nonce = (0, node_crypto_1.randomUUID)();
+        return large ? nonce : nonce.split('-')[0];
+    }
+    generateSendPromise(nonce, message = 'Timeout') {
+        let resolve = (_) => {
+            /**/
+        };
+        const promise = new Promise((res, rej) => {
+            resolve = res;
+            setTimeout(() => {
+                rej(new Error(message));
+            }, 3e3);
+        });
+        this.promises.set(nonce, resolve);
+        return promise;
+    }
+    async send(data, shardId) {
+        const workerId = this.calculateWorkerId(shardId);
+        const worker = this.get(workerId);
+        if (!worker) {
+            throw new Error(`Worker #${workerId} doesnt exist`);
+        }
+        const nonce = this.generateNonce();
+        worker.postMessage({
+            type: 'SEND_PAYLOAD',
+            shardId,
+            nonce,
+            ...data,
+        });
+        return this.generateSendPromise(nonce, 'Shard send payload timeout');
+    }
+    async getShardInfo(shardId) {
+        const workerId = this.calculateWorkerId(shardId);
+        const worker = this.get(workerId);
+        if (!worker) {
+            throw new Error(`Worker #${workerId} doesnt exist`);
+        }
+        const nonce = this.generateNonce(false);
+        worker.postMessage({ shardId, nonce, type: 'SHARD_INFO' });
+        return this.generateSendPromise(nonce, 'Get shard info timeout');
+    }
+    async getWorkerInfo(workerId) {
+        const worker = this.get(workerId);
+        if (!worker) {
+            throw new Error(`Worker #${workerId} doesnt exist`);
+        }
+        const nonce = this.generateNonce();
+        worker.postMessage({ nonce, type: 'WORKER_INFO' });
+        return this.generateSendPromise(nonce, 'Get worker info timeout');
+    }
+    async start() {
+        const spaces = this.prepareSpaces();
+        await this.prepareWorkers(spaces);
+    }
+}
+exports.WorkerManager = WorkerManager;

package/lib/websocket/index.d.ts

@@ -0,0 +1,3 @@
+export * from './SharedTypes';
+export * from './constants';
+export * from './discord';

package/lib/websocket/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./SharedTypes"), exports);
+__exportStar(require("./constants"), exports);
+__exportStar(require("./discord"), exports);

package/lib/websocket/structures/index.d.ts

@@ -0,0 +1,117 @@
+/// <reference types="node" />
+import type { Logger } from '../../common';
+/**
+ * just any kind of request to queue and resolve later
+ */
+export type QueuedRequest = (value: void | Promise<void>) => Promise<unknown> | any;
+/**
+ * options of the dynamic bucket
+ */
+export interface DynamicBucketOptions {
+    limit: number;
+    refillInterval: number;
+    refillAmount: number;
+    debugger?: Logger;
+}
+/**
+ * generally useless for interaction based bots
+ * ideally this would only be triggered on certain paths
+ * example: a huge amount of messages being spammed
+ *
+ * a dynamic bucket is just a priority queue implemented using linked lists
+ * we create an empty bucket for every path
+ * dynamically allocating memory improves the final memory footprint
+ */
+export declare class DynamicBucket {
+    limit: number;
+    refillInterval: number;
+    refillAmount: number;
+    /** The queue of requests to acquire an available request. Mapped by <shardId, resolve()> */
+    queue: PriorityQueue<QueuedRequest>;
+    /** The amount of requests that have been used up already. */
+    used: number;
+    /** Whether or not the queue is already processing. */
+    processing: boolean;
+    /** The timeout id for the timer to reduce the used amount by the refill amount.  */
+    timeoutId?: NodeJS.Timeout;
+    /** The timestamp in milliseconds when the next refill is scheduled. */
+    refillsAt?: number;
+    debugger?: Logger;
+    constructor(options: DynamicBucketOptions);
+    get remaining(): number;
+    refill(): void;
+    /** Begin processing the queue. */
+    processQueue(): Promise<void>;
+    /** Pauses the execution until the request is available to be made. */
+    acquire(priority: number): Promise<void>;
+    toString(): string;
+}
+/**
+ * abstract node lol
+ */
+export interface AbstractNode<T> {
+    data: T;
+    next: this | null;
+}
+export interface QueuePusher<T> {
+    push(data: T): NonNullable<TNode<T>>;
+}
+export interface QueuePusherWithPriority<T> {
+    push(data: T, priority: number): NonNullable<PNode<T>>;
+}
+export declare class TNode<T> implements AbstractNode<T> {
+    data: T;
+    next: this | null;
+    constructor(data: T);
+    static null<T>(list: AbstractNode<T> | null): list is null;
+}
+export declare class PNode<T> extends TNode<T> {
+    priority: number;
+    constructor(data: T, priority: number);
+}
+export declare abstract class Queue<T> {
+    protected abstract head: AbstractNode<T> | null;
+    /**
+     * O(1)
+     */
+    pop(): AbstractNode<T> | null;
+    /**
+     * O(1)
+     */
+    peek(): T;
+    /**
+     * O(n)
+     */
+    size(): number;
+    /**
+     * O(1)
+     */
+    isEmpty(): boolean;
+    [Symbol.iterator](): IterableIterator<T>;
+    toArray(): T[];
+    toString(): string;
+}
+export declare class LinkedList<T> extends Queue<T> implements QueuePusher<T> {
+    protected head: TNode<T> | null;
+    /**
+     * O(1)
+     */
+    push(data: T): NonNullable<TNode<T>>;
+}
+export declare class PriorityQueue<T> extends Queue<T> implements QueuePusherWithPriority<T> {
+    protected head: PNode<T> | null;
+    /**
+     * O(#priorities)
+     */
+    push(data: T, priority: number): NonNullable<PNode<T>>;
+}
+export declare class SequentialBucket {
+    private connections;
+    private capacity;
+    private spawnTimeout;
+    constructor(maxCapacity: number);
+    destroy(): Promise<void>;
+    push(promise: QueuedRequest): Promise<void>;
+    acquire(promises?: LinkedList<QueuedRequest>): Promise<boolean>;
+    static chunk<T>(array: T[], chunks: number): T[][];
+}