rumongo 0.1.0

package/index.js

@@ -0,0 +1,317 @@
+/* tslint:disable */
+/* eslint-disable */
+/* prettier-ignore */
+
+/* auto-generated by NAPI-RS */
+
+const { existsSync, readFileSync } = require('fs')
+const { join } = require('path')
+
+const { platform, arch } = process
+
+let nativeBinding = null
+let localFileExisted = false
+let loadError = null
+
+function isMusl() {
+  // For Node 10
+  if (!process.report || typeof process.report.getReport !== 'function') {
+    try {
+      const lddPath = require('child_process').execSync('which ldd').toString().trim()
+      return readFileSync(lddPath, 'utf8').includes('musl')
+    } catch (e) {
+      return true
+    }
+  } else {
+    const { glibcVersionRuntime } = process.report.getReport().header
+    return !glibcVersionRuntime
+  }
+}
+
+switch (platform) {
+  case 'android':
+    switch (arch) {
+      case 'arm64':
+        localFileExisted = existsSync(join(__dirname, 'rumongo.android-arm64.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./rumongo.android-arm64.node')
+          } else {
+            nativeBinding = require('rumongo-android-arm64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm':
+        localFileExisted = existsSync(join(__dirname, 'rumongo.android-arm-eabi.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./rumongo.android-arm-eabi.node')
+          } else {
+            nativeBinding = require('rumongo-android-arm-eabi')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Android ${arch}`)
+    }
+    break
+  case 'win32':
+    switch (arch) {
+      case 'x64':
+        localFileExisted = existsSync(
+          join(__dirname, 'rumongo.win32-x64-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./rumongo.win32-x64-msvc.node')
+          } else {
+            nativeBinding = require('rumongo-win32-x64-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'ia32':
+        localFileExisted = existsSync(
+          join(__dirname, 'rumongo.win32-ia32-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./rumongo.win32-ia32-msvc.node')
+          } else {
+            nativeBinding = require('rumongo-win32-ia32-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm64':
+        localFileExisted = existsSync(
+          join(__dirname, 'rumongo.win32-arm64-msvc.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./rumongo.win32-arm64-msvc.node')
+          } else {
+            nativeBinding = require('rumongo-win32-arm64-msvc')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Windows: ${arch}`)
+    }
+    break
+  case 'darwin':
+    localFileExisted = existsSync(join(__dirname, 'rumongo.darwin-universal.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./rumongo.darwin-universal.node')
+      } else {
+        nativeBinding = require('rumongo-darwin-universal')
+      }
+      break
+    } catch {}
+    switch (arch) {
+      case 'x64':
+        localFileExisted = existsSync(join(__dirname, 'rumongo.darwin-x64.node'))
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./rumongo.darwin-x64.node')
+          } else {
+            nativeBinding = require('rumongo-darwin-x64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      case 'arm64':
+        localFileExisted = existsSync(
+          join(__dirname, 'rumongo.darwin-arm64.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./rumongo.darwin-arm64.node')
+          } else {
+            nativeBinding = require('rumongo-darwin-arm64')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on macOS: ${arch}`)
+    }
+    break
+  case 'freebsd':
+    if (arch !== 'x64') {
+      throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
+    }
+    localFileExisted = existsSync(join(__dirname, 'rumongo.freebsd-x64.node'))
+    try {
+      if (localFileExisted) {
+        nativeBinding = require('./rumongo.freebsd-x64.node')
+      } else {
+        nativeBinding = require('rumongo-freebsd-x64')
+      }
+    } catch (e) {
+      loadError = e
+    }
+    break
+  case 'linux':
+    switch (arch) {
+      case 'x64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'rumongo.linux-x64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./rumongo.linux-x64-musl.node')
+            } else {
+              nativeBinding = require('rumongo-linux-x64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'rumongo.linux-x64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./rumongo.linux-x64-gnu.node')
+            } else {
+              nativeBinding = require('rumongo-linux-x64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'arm64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'rumongo.linux-arm64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./rumongo.linux-arm64-musl.node')
+            } else {
+              nativeBinding = require('rumongo-linux-arm64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'rumongo.linux-arm64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./rumongo.linux-arm64-gnu.node')
+            } else {
+              nativeBinding = require('rumongo-linux-arm64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'arm':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'rumongo.linux-arm-musleabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./rumongo.linux-arm-musleabihf.node')
+            } else {
+              nativeBinding = require('rumongo-linux-arm-musleabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'rumongo.linux-arm-gnueabihf.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./rumongo.linux-arm-gnueabihf.node')
+            } else {
+              nativeBinding = require('rumongo-linux-arm-gnueabihf')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 'riscv64':
+        if (isMusl()) {
+          localFileExisted = existsSync(
+            join(__dirname, 'rumongo.linux-riscv64-musl.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./rumongo.linux-riscv64-musl.node')
+            } else {
+              nativeBinding = require('rumongo-linux-riscv64-musl')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        } else {
+          localFileExisted = existsSync(
+            join(__dirname, 'rumongo.linux-riscv64-gnu.node')
+          )
+          try {
+            if (localFileExisted) {
+              nativeBinding = require('./rumongo.linux-riscv64-gnu.node')
+            } else {
+              nativeBinding = require('rumongo-linux-riscv64-gnu')
+            }
+          } catch (e) {
+            loadError = e
+          }
+        }
+        break
+      case 's390x':
+        localFileExisted = existsSync(
+          join(__dirname, 'rumongo.linux-s390x-gnu.node')
+        )
+        try {
+          if (localFileExisted) {
+            nativeBinding = require('./rumongo.linux-s390x-gnu.node')
+          } else {
+            nativeBinding = require('rumongo-linux-s390x-gnu')
+          }
+        } catch (e) {
+          loadError = e
+        }
+        break
+      default:
+        throw new Error(`Unsupported architecture on Linux: ${arch}`)
+    }
+    break
+  default:
+    throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`)
+}
+
+if (!nativeBinding) {
+  if (loadError) {
+    throw loadError
+  }
+  throw new Error(`Failed to load native binding`)
+}
+
+const { MongoClient, FindCursor, RawDoc } = nativeBinding
+
+module.exports.MongoClient = MongoClient
+module.exports.FindCursor = FindCursor
+module.exports.RawDoc = RawDoc

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "rumongo",
+  "version": "0.1.0",
+  "description": "Rust-native MongoDB read driver for Node.js — faster reads than the official driver and Mongoose",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "license": "MIT",
+  "author": "Piyush Bhangale",
+  "keywords": [
+    "mongodb",
+    "mongo",
+    "driver",
+    "rust",
+    "napi",
+    "bson",
+    "mongoose",
+    "performance",
+    "native"
+  ],
+  "engines": {
+    "node": ">=16"
+  },
+  "napi": {
+    "name": "rumongo"
+  },
+  "files": [
+    "index.js",
+    "index.d.ts",
+    "dist/",
+    "worker/",
+    "rumongo.*.node",
+    "MIGRATION.md",
+    "BENCHMARKS.md",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "napi build --platform --release",
+    "build:debug": "napi build --platform",
+    "build:ts": "tsc",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "napi build --platform --release && tsc",
+    "test:integration": "node --test __tests__/integration/"
+  },
+  "devDependencies": {
+    "@napi-rs/cli": "^2.18.4",
+    "mongodb": "^6.8.0",
+    "mongoose": "^8.5.0",
+    "typescript": "^5.4.5"
+  }
+}

package/worker/pool-worker.js

@@ -0,0 +1,47 @@
+// Worker-side entry for the rumongo worker pool.
+// Loads the native addon, holds its own MongoClient, answers find requests.
+// Runs on its OWN V8 isolate, so BSON->JS materialization here does not block
+// the main event loop.
+
+const { parentPort, workerData } = require('worker_threads')
+const { MongoClient } = require('../index.js')
+
+let client
+
+async function handle(msg) {
+  const { id, op, db, coll, filter, opts } = msg
+  try {
+    if (op === 'close') {
+      if (client) await client.close()
+      parentPort.postMessage({ id, ok: true })
+      return
+    }
+    if (op === 'find') {
+      // Return one JSON string per doc (same shape the addon returns). The
+      // worker did the fetch + BSON decode on this isolate; main only receives.
+      const rows = await client.find(db, coll, filter, opts)
+      parentPort.postMessage({ id, rows })
+      return
+    }
+    if (op === 'reduce') {
+      // Generic "do the work in the worker" path: fetch + parse + run the
+      // caller's reducer here, return only the accumulated result to main.
+      // reducerSource is a function source string: (acc, doc) => acc
+      const reducer = (0, eval)(`(${msg.reducerSource})`)
+      const rows = await client.find(db, coll, filter, opts)
+      let acc = msg.init
+      for (const r of rows) acc = reducer(acc, JSON.parse(r))
+      parentPort.postMessage({ id, acc, count: rows.length })
+      return
+    }
+    parentPort.postMessage({ id, error: `unknown op ${op}` })
+  } catch (e) {
+    parentPort.postMessage({ id, error: e.message })
+  }
+}
+
+;(async () => {
+  client = await MongoClient.connect(workerData.uri)
+  parentPort.on('message', handle)
+  parentPort.postMessage({ ready: true })
+})().catch((e) => parentPort.postMessage({ fatal: e.message }))

package/worker/pool.d.ts

@@ -0,0 +1,42 @@
+// Type declarations for the opt-in worker pool (worker/pool.js).
+
+export interface WorkerPoolOptions {
+  /** MongoDB connection string. */
+  uri: string
+  /** Number of worker threads. Default: cpus - 2 (min 2). */
+  size?: number
+}
+
+/**
+ * Opt-in pool of Node worker threads, each running the rumongo addon on its
+ * own V8 isolate. Use `reduce` to run work in a worker and keep the main event
+ * loop free. `find` returns rows to main (no jitter benefit — prefer the direct
+ * client for that).
+ */
+export declare class WorkerPool {
+  static create(opts: WorkerPoolOptions): Promise<WorkerPool>
+
+  /** Query in a worker, return parsed rows to main (data crosses the boundary). */
+  find<T = Record<string, unknown>>(
+    db: string,
+    coll: string,
+    filter?: object,
+    opts?: object,
+  ): Promise<T[]>
+
+  /**
+   * Run a reducer over the query results INSIDE a worker; only the accumulator
+   * crosses back to main. The reducer is shipped as source, so it must be pure
+   * (no closure over main-thread variables): `(acc, doc) => acc`.
+   */
+  reduce<A>(
+    db: string,
+    coll: string,
+    filter: object,
+    opts: object,
+    reducer: (acc: A, doc: Record<string, unknown>) => A,
+    init: A,
+  ): Promise<{ acc: A; count: number }>
+
+  close(): Promise<void>
+}

package/worker/pool.js

@@ -0,0 +1,100 @@
+// Main-side worker pool for rumongo.
+// Spawns N Node worker threads (each loads the addon + its own MongoClient),
+// round-robins requests, matches replies by id. Keeps the main event loop free
+// of BSON->JS work for the queries it dispatches.
+
+const { Worker } = require('worker_threads')
+const path = require('path')
+
+class WorkerPool {
+  constructor() {
+    this._workers = []
+    this._next = 0
+    this._seq = 0
+    this._pending = new Map() // id -> {resolve, reject}
+  }
+
+  static async create({ uri, size = Math.max(2, require('os').cpus().length - 2) } = {}) {
+    const pool = new WorkerPool()
+    await Promise.all(
+      Array.from({ length: size }, () => pool._spawn(uri)),
+    )
+    return pool
+  }
+
+  _spawn(uri) {
+    return new Promise((resolve, reject) => {
+      const w = new Worker(path.join(__dirname, 'pool-worker.js'), { workerData: { uri } })
+      w.on('message', (msg) => {
+        if (msg.ready) return resolve()
+        if (msg.fatal) return reject(new Error(msg.fatal))
+        const p = this._pending.get(msg.id)
+        if (!p) return
+        this._pending.delete(msg.id)
+        if (msg.error) p.reject(new Error(msg.error))
+        else p.resolve(msg)
+      })
+      w.on('error', reject)
+      this._workers.push(w)
+    })
+  }
+
+  _dispatch(payload) {
+    const id = ++this._seq
+    const w = this._workers[this._next++ % this._workers.length]
+    return new Promise((resolve, reject) => {
+      this._pending.set(id, { resolve, reject })
+      w.postMessage({ id, ...payload })
+    })
+  }
+
+  /** find -> array of parsed JS objects (main parses the JSON strings). */
+  async find(db, coll, filter = {}, opts = {}) {
+    const msg = await this._dispatch({
+      op: 'find',
+      db,
+      coll,
+      filter: JSON.stringify(filter),
+      opts: JSON.stringify(opts),
+    })
+    return msg.rows.map((r) => JSON.parse(r))
+  }
+
+  /**
+   * Run a reducer over the query results INSIDE a worker; only the accumulated
+   * result crosses back to main. The reducer must be a pure function with no
+   * closure over main-thread state (it is shipped as source and rebuilt in the
+   * worker): `(acc, doc) => acc`.
+   *
+   *   await pool.reduce('db','users', {active:true}, {}, (a,d)=>a+d.age, 0)
+   */
+  async reduce(db, coll, filter, opts, reducer, init) {
+    const msg = await this._dispatch({
+      op: 'reduce',
+      db,
+      coll,
+      filter: JSON.stringify(filter || {}),
+      opts: JSON.stringify(opts || {}),
+      reducerSource: reducer.toString(),
+      init,
+    })
+    return { acc: msg.acc, count: msg.count }
+  }
+
+  async close() {
+    // Send a close to EACH worker (not round-robin), then terminate.
+    await Promise.all(
+      this._workers.map(
+        (w) =>
+          new Promise((resolve) => {
+            const id = ++this._seq
+            this._pending.set(id, { resolve, reject: resolve })
+            w.postMessage({ id, op: 'close' })
+          }),
+      ),
+    )
+    await Promise.all(this._workers.map((w) => w.terminate()))
+  }
+}
+
+module.exports = { WorkerPool }