prisma-sharding 0.0.1 → 0.0.3

package/README.md

@@ -0,0 +1,330 @@
+# Prisma Sharding
+
+Lightweight database sharding library for Prisma with connection pooling, health monitoring, and CLI tools.
+
+## Installation
+
+```bash
+yarn add prisma-sharding
+# or
+npm install prisma-sharding
+```
+
+> Don't forget to follow me on [GitHub](https://github.com/safdar-azeem)!
+
+## Step 1: Create Sharding Connection
+
+```typescript
+// src/config/prisma.ts
+
+import { PrismaSharding } from 'prisma-sharding';
+import { PrismaClient } from '@/generated/prisma';
+import { PrismaPg } from '@prisma/adapter-pg';
+
+const sharding = new PrismaSharding<PrismaClient>({
+  shards: [
+    { id: 'shard_1', url: process.env.SHARD_1_URL! },
+    { id: 'shard_2', url: process.env.SHARD_2_URL! },
+  ],
+  strategy: 'modulo', // 'modulo' | 'consistent-hash'
+  createClient: (url) => {
+    const adapter = new PrismaPg({ connectionString: url, max: 10 });
+    return new PrismaClient({ adapter });
+  },
+});
+
+await sharding.connect();
+```
+
+## API
+
+| Method                       | Description                                   |
+| ---------------------------- | --------------------------------------------- |
+| `getShard(key)`              | Get Prisma client for a given key             |
+| `getShardById(shardId)`      | Get Prisma client by shard ID                 |
+| `getRandomShard()`           | Get random shard (for new records)            |
+| `findFirst(fn)`              | Search across all shards, return first result |
+| `runOnAll(fn)`               | Execute on all shards                         |
+| `getHealth()`                | Get health status of all shards               |
+| `connect()` / `disconnect()` | Lifecycle methods                             |
+
+### Step 2: Create a User (Assign to a Shard)
+
+New records should be created on a random shard for even distribution.
+
+```ts
+import { sharding } from '@/config/prisma';
+
+const client = sharding.getRandomShard();
+
+const user = await client.user.create({
+  data: {
+    email: 'user@example.com',
+    username: 'new_user',
+  },
+});
+```
+
+## Step 3: Access User by ID (Shard Routing)
+
+When you have a user ID, Prisma Sharding routes you to the correct shard automatically.
+
+```ts
+const userId = 'abc123';
+
+const client = sharding.getShard(userId);
+
+const user = await client.user.findUnique({
+  where: { id: userId },
+});
+```
+
+`Important rule:`
+
+Once you get the shard client using a user ID, **all future operations for that user must use this same client**.
+
+That includes:
+
+- Reading user data
+- Updating user data
+- Creating related records (profiles, posts, settings, etc)
+
+Every user belongs to exactly one shard. Their entire data lives on that shard only.
+
+Do **not** switch shards or use a random shard for user related actions.
+
+Always do this:
+
+```ts
+const client = sharding.getShard(userId);
+```
+
+This guarantees all user data stays on the correct shard and avoids cross shard bugs.
+
+### Step 4: Find User Without ID (Cross Shard Search)
+
+If you do not have the user ID, search all shards in parallel.
+Use this only when necessary.
+
+```typescript
+// Find user by email across ALL shards (parallel execution)
+const { result: user, client } = await sharding.findFirst(async (c) =>
+  c.user.findFirst({ where: { email } })
+);
+
+if (user && client) {
+  // Continue operations on the found shard
+  await client.user.update({
+    where: { id: user.id },
+    data: { lastLogin: new Date() },
+  });
+}
+```
+
+## Step 5: Run on All Shards (Admin or Analytics)
+
+```typescript
+// Get counts from all shards
+const counts = await sharding.runOnAll(async (client) => client.user.count());
+const totalUsers = counts.reduce((sum, count) => sum + count, 0);
+
+// With detailed results (includes errors)
+const results = await sharding.runOnAllWithDetails(async (client, shardId) => {
+  return { shardId, count: await client.user.count() };
+});
+```
+
+### Health Monitoring
+
+```typescript
+// Get health of all shards
+const health = sharding.getHealth();
+// Returns: [{ shardId, isHealthy, latencyMs, lastChecked, ... }]
+
+// Get specific shard health
+const shard1Health = sharding.getHealthByShard('shard_1');
+```
+
+### Lifecycle
+
+```typescript
+// Graceful shutdown
+await sharding.disconnect();
+
+// Check connection status
+if (sharding.isConnected()) {
+  // ...
+}
+```
+
+## CLI Tools
+
+The package includes CLI tools for common sharding operations. No need to write custom scripts!
+
+### Setup
+
+Add to your `package.json`:
+
+```json
+{
+  "scripts": {
+    "db:studio:all": "prisma-sharding-studio",
+    "migrate:shards": "prisma-sharding-migrate",
+    "test:shards": "prisma-sharding-test"
+  }
+}
+```
+
+### Environment Variables
+
+```bash
+SHARD_COUNT=3
+SHARD_1_URL=postgresql://user:pass@host:5432/db1
+SHARD_2_URL=postgresql://user:pass@host:5432/db2
+SHARD_3_URL=postgresql://user:pass@host:5432/db3
+SHARD_ROUTING_STRATEGY=modulo  # or consistent-hash
+SHARD_STUDIO_BASE_PORT=51212   # optional, for studio
+```
+
+### Commands
+
+#### `prisma-sharding-migrate`
+
+Push schema to all shards using `prisma db push`.
+
+```bash
+yarn migrate:shards
+```
+
+#### `prisma-sharding-studio`
+
+Start Prisma Studio for all shards on sequential ports.
+
+```bash
+yarn db:studio:all
+# Opens shard_1 on :51212, shard_2 on :51213, etc.
+```
+
+#### `prisma-sharding-test`
+
+Test connections to all shards.
+
+```bash
+yarn test:shards
+```
+
+## Configuration
+
+| Option                    | Type                            | Default    | Description                       |
+| ------------------------- | ------------------------------- | ---------- | --------------------------------- |
+| `shards`                  | `ShardConfig[]`                 | Required   | Array of shard configurations     |
+| `strategy`                | `'modulo' \| 'consistent-hash'` | `'modulo'` | Routing algorithm                 |
+| `createClient`            | `(url, shardId) => TClient`     | Required   | Factory to create Prisma clients  |
+| `healthCheckIntervalMs`   | `number`                        | `30000`    | Health check frequency            |
+| `circuitBreakerThreshold` | `number`                        | `3`        | Failures before marking unhealthy |
+
+### Shard Config
+
+```typescript
+interface ShardConfig {
+  id: string; // Unique identifier (e.g., 'shard_1')
+  url: string; // PostgreSQL connection string
+  weight?: number; // Optional weight for distribution
+  isReadReplica?: boolean;
+}
+```
+
+## Routing Strategies
+
+### Modulo (Default)
+
+Simple and fast. Uses `hash(key) % shardCount` for routing.
+
+```typescript
+strategy: 'modulo';
+```
+
+### Consistent Hash
+
+Minimizes data movement when adding/removing shards.
+
+```typescript
+strategy: 'consistent-hash';
+```
+
+## Error Handling
+
+```typescript
+import { ShardingError, ConfigError, ConnectionError } from 'prisma-sharding';
+
+try {
+  const client = sharding.getShard(userId);
+} catch (error) {
+  if (error instanceof ConnectionError) {
+    console.error(`Shard ${error.shardId} unavailable`);
+  }
+}
+```
+
+## Custom Logger
+
+```typescript
+const sharding = new PrismaSharding({
+  // ...config,
+  logger: {
+    info: (msg) => myLogger.info(msg),
+    warn: (msg) => myLogger.warn(msg),
+    error: (msg) => myLogger.error(msg),
+  },
+});
+```
+
+---
+
+### `getAllClients()`
+
+Get all Prisma client instances.
+
+```typescript
+const clients = sharding.getAllClients();
+
+console.log(`Managing ${clients.length} shard clients`);
+```
+
+**Returns:** `PrismaClient[]`
+
+---
+
+### `getShardCount()`
+
+Get total number of configured shards.
+
+```typescript
+const count = sharding.getShardCount();
+console.log(`Running on ${count} shards`);
+// Output: Running on 3 shards
+```
+
+---
+
+### `getShardIds()`
+
+Get array of all shard IDs.
+
+```typescript
+const shardIds = sharding.getShardIds();
+console.log(shardIds);
+// Output: ['shard_1', 'shard_2', 'shard_3']
+```
+
+**Returns:** `string[]`
+
+---
+
+## Author
+
+[safdar-azeem](https://github.com/safdar-azeem)
+
+## License
+
+MIT

package/dist/cli/migrate.js

@@ -0,0 +1,117 @@
+#!/usr/bin/env node
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+
+// src/cli/migrate.ts
+var import_config = require("dotenv/config");
+var import_child_process = require("child_process");
+var import_path = __toESM(require("path"));
+var getShardConfigs = () => {
+  const shards = [];
+  const shardCount = parseInt(process.env.SHARD_COUNT || "0", 10);
+  for (let i = 1; i <= shardCount; i++) {
+    const url = process.env[`SHARD_${i}_URL`];
+    if (url) {
+      shards.push({ id: `shard_${i}`, url });
+    }
+  }
+  if (shards.length === 0 && process.env.DATABASE_URL) {
+    shards.push({ id: "shard_1", url: process.env.DATABASE_URL });
+  }
+  return shards;
+};
+var runPrismaCommand = (shardUrl, command) => {
+  return new Promise((resolve) => {
+    const prisma = (0, import_child_process.spawn)("npx", ["prisma", ...command, "--url", shardUrl], {
+      env: process.env,
+      cwd: import_path.default.resolve(process.cwd()),
+      shell: true
+    });
+    let output = "";
+    let errorOutput = "";
+    prisma.stdout.on("data", (data) => {
+      output += data.toString();
+    });
+    prisma.stderr.on("data", (data) => {
+      errorOutput += data.toString();
+    });
+    prisma.on("close", (code) => {
+      if (code === 0) {
+        resolve({ output });
+      } else {
+        resolve({ output, error: errorOutput || `Exit code: ${code}` });
+      }
+    });
+    prisma.on("error", (err) => {
+      resolve({ output, error: err.message });
+    });
+  });
+};
+var migrateAllShards = async () => {
+  const shards = getShardConfigs();
+  console.log("\u{1F504} prisma-sharding: Starting migrations...\n");
+  console.log(`\u{1F4CA} Total shards to migrate: ${shards.length}
+`);
+  if (shards.length === 0) {
+    console.error(
+      "\u274C No shards configured. Set SHARD_COUNT and SHARD_N_URL environment variables."
+    );
+    process.exit(1);
+  }
+  const results = [];
+  for (const shard of shards) {
+    console.log(`
+\u{1F4E6} Migrating ${shard.id}...`);
+    console.log(`   URL: ${shard.url.replace(/:[^:@]+@/, ":***@")}`);
+    const { output, error } = await runPrismaCommand(shard.url, ["db", "push"]);
+    if (error && !output.includes("Your database is now in sync")) {
+      console.error(`   \u274C Failed: ${error.split("\n")[0]}`);
+      results.push({ shardId: shard.id, success: false, output, error });
+    } else {
+      console.log(`   \u2705 Success`);
+      results.push({ shardId: shard.id, success: true, output });
+    }
+  }
+  console.log("\n" + "=".repeat(50));
+  console.log("\u{1F4CB} Migration Summary\n");
+  const successful = results.filter((r) => r.success).length;
+  const failed = results.filter((r) => !r.success).length;
+  results.forEach((result) => {
+    const status = result.success ? "\u2705" : "\u274C";
+    console.log(
+      `   ${status} ${result.shardId}${result.error ? ` - ${result.error.split("\n")[0]}` : ""}`
+    );
+  });
+  console.log(`
+   Total: ${results.length} | Success: ${successful} | Failed: ${failed}`);
+  if (failed > 0) {
+    console.log("\n\u26A0\uFE0F  Some migrations failed. Please review the errors above.");
+    process.exit(1);
+  }
+  console.log("\n\u2705 All shard migrations completed successfully!");
+};
+migrateAllShards().catch((error) => {
+  console.error("Migration script failed:", error);
+  process.exit(1);
+});

package/dist/cli/studio.js

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+"use strict";
+
+// src/cli/studio.ts
+var import_config = require("dotenv/config");
+var import_child_process = require("child_process");
+var instances = [];
+var BASE_PORT = parseInt(process.env.SHARD_STUDIO_BASE_PORT || "51212", 10);
+var getShardConfigs = () => {
+  const shards = [];
+  const shardCount = parseInt(process.env.SHARD_COUNT || "0", 10);
+  for (let i = 1; i <= shardCount; i++) {
+    const url = process.env[`SHARD_${i}_URL`];
+    if (url) {
+      shards.push({ id: `shard_${i}`, url });
+    }
+  }
+  if (shards.length === 0 && process.env.DATABASE_URL) {
+    shards.push({ id: "shard_1", url: process.env.DATABASE_URL });
+  }
+  return shards;
+};
+var startStudio = (shard, index) => {
+  return new Promise((resolve, reject) => {
+    const port = BASE_PORT + index;
+    const shardId = shard.id;
+    console.log(`
+\u{1F680} Starting Prisma Studio for ${shardId} on port ${port}...`);
+    console.log(`   URL: ${shard.url.replace(/:[^:@]+@/, ":***@")}`);
+    const studioProcess = (0, import_child_process.spawn)(
+      "npx",
+      ["prisma", "studio", "--port", port.toString(), "--browser", "none"],
+      {
+        env: {
+          ...process.env,
+          DATABASE_URL: shard.url
+        },
+        shell: true,
+        stdio: "pipe"
+      }
+    );
+    studioProcess.stdout?.on("data", (data) => {
+      const output = data.toString();
+      if (output.includes("Prisma Studio is running")) {
+        console.log(`   \u2705 ${shardId} ready at http://localhost:${port}`);
+      }
+    });
+    studioProcess.stderr?.on("data", (data) => {
+      const output = data.toString();
+      if (!output.includes("warn") && !output.includes("Loaded")) {
+        console.error(`   [${shardId}] ${output}`);
+      }
+    });
+    studioProcess.on("error", (err) => {
+      console.error(`   \u274C Failed to start ${shardId}:`, err.message);
+      reject(err);
+    });
+    const instance = {
+      shardId,
+      port,
+      process: studioProcess
+    };
+    instances.push(instance);
+    setTimeout(() => resolve(instance), 2e3);
+  });
+};
+var startAllStudios = async () => {
+  const shards = getShardConfigs();
+  console.log("=".repeat(60));
+  console.log("\u{1F5C4}\uFE0F  prisma-sharding: Multi-Shard Studio Viewer");
+  console.log("=".repeat(60));
+  console.log(`
+\u{1F4CA} Starting ${shards.length} Prisma Studio instance(s)...
+`);
+  if (shards.length === 0) {
+    console.error(
+      "\u274C No shards configured. Set SHARD_COUNT and SHARD_N_URL environment variables."
+    );
+    process.exit(1);
+  }
+  for (let i = 0; i < shards.length; i++) {
+    try {
+      await startStudio(shards[i], i);
+    } catch (error) {
+      console.error(`Failed to start studio for ${shards[i].id}:`, error);
+    }
+  }
+  console.log("\n" + "=".repeat(60));
+  console.log("\u{1F4CB} All Studios Running:");
+  console.log("=".repeat(60));
+  instances.forEach((instance) => {
+    console.log(`   \u2022 ${instance.shardId}: http://localhost:${instance.port}`);
+  });
+  console.log("\n   Press Ctrl+C to stop all instances\n");
+};
+var gracefulShutdown = () => {
+  console.log("\n\n\u{1F6D1} Shutting down all Prisma Studio instances...\n");
+  instances.forEach((instance) => {
+    console.log(`   Stopping ${instance.shardId}...`);
+    instance.process.kill("SIGTERM");
+  });
+  setTimeout(() => {
+    console.log("\n\u2705 All instances stopped\n");
+    process.exit(0);
+  }, 1e3);
+};
+process.on("SIGINT", gracefulShutdown);
+process.on("SIGTERM", gracefulShutdown);
+startAllStudios().catch((error) => {
+  console.error("Failed to start studios:", error);
+  process.exit(1);
+});

package/dist/cli/test.js

@@ -0,0 +1,122 @@
+#!/usr/bin/env node
+"use strict";
+
+// src/cli/test.ts
+var import_config = require("dotenv/config");
+var import_child_process = require("child_process");
+var results = [];
+var log = {
+  info: (msg) => console.log(`\u2139\uFE0F  ${msg}`),
+  success: (msg) => console.log(`\u2705 ${msg}`),
+  error: (msg) => console.log(`\u274C ${msg}`),
+  warn: (msg) => console.log(`\u26A0\uFE0F  ${msg}`),
+  section: (msg) => console.log(`
+${"=".repeat(50)}
+\u{1F4CB} ${msg}
+${"=".repeat(50)}`)
+};
+var getShardConfigs = () => {
+  const shards = [];
+  const shardCount = parseInt(process.env.SHARD_COUNT || "0", 10);
+  for (let i = 1; i <= shardCount; i++) {
+    const url = process.env[`SHARD_${i}_URL`];
+    if (url) {
+      shards.push({ id: `shard_${i}`, url });
+    }
+  }
+  if (shards.length === 0 && process.env.DATABASE_URL) {
+    shards.push({ id: "shard_1", url: process.env.DATABASE_URL });
+  }
+  return shards;
+};
+var runTest = async (name, testFn) => {
+  const start = Date.now();
+  try {
+    await testFn();
+    const duration = Date.now() - start;
+    results.push({ name, passed: true, message: "Passed", duration });
+    log.success(`${name} (${duration}ms)`);
+  } catch (error) {
+    const duration = Date.now() - start;
+    const message = error instanceof Error ? error.message : String(error);
+    results.push({ name, passed: false, message, duration });
+    log.error(`${name}: ${message}`);
+  }
+};
+var testConnection = (url) => {
+  return new Promise((resolve) => {
+    const prisma = (0, import_child_process.spawn)("npx", ["prisma", "db", "execute", "--url", url, "--stdin"], {
+      shell: true,
+      stdio: ["pipe", "pipe", "pipe"]
+    });
+    prisma.stdin.write("SELECT 1");
+    prisma.stdin.end();
+    prisma.on("close", (code) => {
+      resolve(code === 0);
+    });
+    prisma.on("error", () => {
+      resolve(false);
+    });
+  });
+};
+var runTests = async () => {
+  const shards = getShardConfigs();
+  console.log("\n\u{1F9EA} prisma-sharding: Shard Connection Test Suite\n");
+  console.log(`\u{1F4CA} Configuration:`);
+  console.log(`   - Shard Count: ${shards.length}`);
+  console.log(`   - Routing Strategy: ${process.env.SHARD_ROUTING_STRATEGY || "modulo"}`);
+  if (shards.length === 0) {
+    console.error(
+      "\n\u274C No shards configured. Set SHARD_COUNT and SHARD_N_URL environment variables."
+    );
+    process.exit(1);
+  }
+  log.section("Test 1: Shard Connection Tests");
+  for (const shard of shards) {
+    await runTest(`Connect to ${shard.id}`, async () => {
+      const success = await testConnection(shard.url);
+      if (!success) {
+        throw new Error(`Failed to connect to ${shard.id}`);
+      }
+    });
+  }
+  log.section("Test 2: Configuration Verification");
+  await runTest("Verify environment variables", async () => {
+    const shardCount = parseInt(process.env.SHARD_COUNT || "0", 10);
+    if (shardCount === 0 && !process.env.DATABASE_URL) {
+      throw new Error("SHARD_COUNT or DATABASE_URL must be set");
+    }
+    log.info(`SHARD_COUNT: ${shardCount}`);
+    log.info(`Routing Strategy: ${process.env.SHARD_ROUTING_STRATEGY || "modulo"}`);
+  });
+  await runTest("Verify all shard URLs are present", async () => {
+    const shardCount = parseInt(process.env.SHARD_COUNT || "0", 10);
+    for (let i = 1; i <= shardCount; i++) {
+      const url = process.env[`SHARD_${i}_URL`];
+      if (!url) {
+        throw new Error(`SHARD_${i}_URL is missing`);
+      }
+    }
+  });
+  log.section("Test Results Summary");
+  const passed = results.filter((r) => r.passed).length;
+  const failed = results.filter((r) => !r.passed).length;
+  const totalDuration = results.reduce((sum, r) => sum + r.duration, 0);
+  console.log(`
+   Total Tests: ${results.length}`);
+  console.log(`   Passed: ${passed}`);
+  console.log(`   Failed: ${failed}`);
+  console.log(`   Duration: ${totalDuration}ms`);
+  if (failed > 0) {
+    console.log("\n   Failed Tests:");
+    results.filter((r) => !r.passed).forEach((r) => {
+      console.log(`     - ${r.name}: ${r.message}`);
+    });
+    process.exit(1);
+  }
+  console.log("\n\u2705 All shard tests passed!");
+};
+runTests().catch((error) => {
+  console.error("\n\u{1F4A5} Test suite crashed:", error);
+  process.exit(1);
+});