sails-sqlite 0.0.0 → 0.1.0

package/lib/private/build-std-adapter-method.js

@@ -0,0 +1,65 @@
+const Machine = require('machine')
+
+/**
+ * buildStdAdapterMethod()
+ *
+ * Build a generic DQL/DML adapter method for SQLite from a machine definition and available state.
+ *
+ * @param {Object} machineDef - The machine definition (dry)
+ * @param {Object} registeredDsEntries - Registered datastore entries
+ * @param {Object} registeredDryModels - Registered dry models
+ * @returns {Function} - The adapter method
+ */
+module.exports = function buildStdAdapterMethod(
+  machineDef,
+  wetMachines,
+  registeredDsEntries,
+  registeredDryModels
+) {
+  // Build wet machine.
+  const performQuery = Machine.build(machineDef)
+
+  // Return function that will be the adapter method.
+  return function (datastoreName, s3q, done) {
+    // Look up the datastore entry (to get the manager).
+    const dsEntry = registeredDsEntries[datastoreName]
+
+    // Sanity check:
+    if (!dsEntry) {
+      return done(
+        new Error(
+          `Consistency violation: Cannot do that with datastore (${datastoreName}) because no matching datastore entry is registered in this adapter! This is usually due to a race condition (e.g. a lifecycle callback still running after the ORM has been torn down), or it could be due to a bug in this adapter. (If you get stumped, reach out at http://sailsjs.com/support.)`
+        )
+      )
+    }
+
+    // For SQLite, we don't need to obtain a separate connection. The manager is the connection.
+    const connection = dsEntry.manager
+
+    // Build switch handlers based on the machine's defined exits
+    const switchHandlers = {
+      error: function (err) {
+        return done(err)
+      },
+      success: function (result) {
+        return done(null, result)
+      }
+    }
+
+    // Only add notUnique handler if the machine defines this exit
+    if (machineDef.exits.notUnique) {
+      switchHandlers.notUnique = function (err) {
+        return done(
+          Object.assign(new Error('Not unique'), { code: 'E_UNIQUE' })
+        )
+      }
+    }
+
+    // Perform the query
+    performQuery({
+      query: s3q,
+      connection: connection,
+      dryOrm: { models: registeredDryModels }
+    }).switch(switchHandlers)
+  }
+}

package/lib/private/constants/connection.input.js

@@ -0,0 +1,15 @@
+/**
+ * `connection`
+ *
+ * @constant
+ * @type {InputDef}
+ */
+module.exports = {
+  description: 'The active database connection to use.',
+  extendedDescription:
+    'This connection _will not be released automatically_ or mutated in any other way by this machine.',
+  whereToGet: { description: 'Use getConnection().' },
+  example: '===',
+  readOnly: true,
+  required: true
+}

package/lib/private/constants/dry-orm.input.js

@@ -0,0 +1,23 @@
+/**
+ * `dryOrm`
+ *
+ * @constant
+ * @type {InputDef}
+ */
+module.exports = {
+  friendlyName: 'Dry ORM',
+  description: 'The "dry ORM" instance.',
+  extendedDescription:
+    'This includes a property called `models`, which is a dictionary containing all known model definitions, keyed by model identity.',
+  required: true,
+  readOnly: true,
+  example: '==='
+  //e.g.
+  //```
+  //{
+  //  models: {
+  //    pet: {attributes:{...}, tableName: 'sack_of_pets', identity: 'pet'},
+  //  },
+  //}
+  //```
+}

package/lib/private/constants/meta.input.js

@@ -0,0 +1,14 @@
+/**
+ * `meta`
+ *
+ * @constant
+ * @type {InputDef}
+ */
+module.exports = {
+  friendlyName: 'Meta (custom)',
+  description:
+    "A dictionary of additional options to customize this behavior. (e.g. `{foo: 'bar'}`)",
+  moreInfoUrl:
+    'https://github.com/node-machine/driver-interface/blob/3f3a150ef4ece40dc0d105006e2766e81af23719/constants/meta.input.js',
+  example: '==='
+}

package/lib/private/constants/not-unique.exit.js

@@ -0,0 +1,16 @@
+/**
+ * `notUnique`
+ *
+ * @constant
+ * @type {ExitDef}
+ */
+module.exports = {
+  description:
+    'Could not persist changes because they would violate one or more uniqueness constraints.',
+  moreInfoUrl:
+    'https://github.com/sailshq/waterline-query-docs/blob/8fc158d8460aa04ee6233fefbdf83cc17e7645df/docs/errors.md',
+  outputFriendlyName: 'Uniqueness error',
+  outputDescription:
+    'A native error from the database, with an extra key (`footprint`) attached.',
+  outputExample: '===' // e.g. an Error instance with a `footprint` attached
+}

package/lib/private/constants/query.input.js

@@ -0,0 +1,15 @@
+/**
+ * `query`
+ *
+ * @constant
+ * @type {InputDef}
+ */
+module.exports = {
+  friendlyName: 'Query (s3q)',
+  description: 'A stage three Waterline query.',
+  extendedDescription:
+    'The `meta` key of this dictionary is reserved for certain special "meta keys" (e.g. flags, signals, etc.) and other custom, adapter-specific extensions.',
+  required: true,
+  readOnly: true,
+  example: '===' //e.g. `{ method: 'create', using: 'the_table_name', ... }`
+}

package/lib/private/constants/table-name.input.js

@@ -0,0 +1,12 @@
+/**
+ * `tableName`
+ *
+ * @constant
+ * @type {InputDef}
+ */
+module.exports = {
+  friendlyName: 'Table name',
+  description:
+    'The name of the physical model (i.e. the name of the Mongo collection -- aka "tableName").',
+  example: 'foo_bar'
+}

package/lib/private/machines/avg-records.js

@@ -0,0 +1,74 @@
+const buildSqliteWhereClause = require('./private/build-sqlite-where-clause')
+
+module.exports = {
+  friendlyName: 'Avg (records)',
+
+  description: 'Return the Average of the records matched by the query.',
+
+  inputs: {
+    query: require('../constants/query.input'),
+    connection: require('../constants/connection.input'),
+    dryOrm: require('../constants/dry-orm.input')
+  },
+
+  exits: {
+    success: {
+      outputFriendlyName: 'Average (mean)',
+      outputDescription:
+        'The average value of the given property across all records.',
+      outputExample: -48.1293
+    }
+  },
+
+  fn: function (inputs, exits) {
+    const s3q = inputs.query
+
+    const tableName = s3q.using
+    const numericFieldName = s3q.numericAttrName
+
+    // Grab the model definition
+    // Find model by tableName since models is an object, not an array
+    let WLModel = null
+    for (const modelIdentity in inputs.dryOrm.models) {
+      if (inputs.dryOrm.models[modelIdentity].tableName === tableName) {
+        WLModel = inputs.dryOrm.models[modelIdentity]
+        break
+      }
+    }
+    if (!WLModel) {
+      return exits.error(
+        new Error(
+          `No model with that tableName (\`${tableName}\`) has been registered with this adapter. Were any unexpected modifications made to the stage 3 query? Could the adapter's internal state have been corrupted? (This error is usually due to a bug in this adapter's implementation.)`
+        )
+      )
+    }
+
+    // Build a SQLite WHERE clause from the `where` clause.
+    let whereClause
+    try {
+      whereClause = buildSqliteWhereClause(
+        s3q.criteria.where,
+        WLModel,
+        s3q.meta
+      )
+    } catch (e) {
+      return exits.error(e)
+    }
+
+    const db = inputs.connection
+
+    try {
+      let avgQuery = `SELECT COALESCE(AVG(${numericFieldName}), 0) as average FROM ${tableName}`
+      if (whereClause) {
+        avgQuery += ` WHERE ${whereClause}`
+      }
+
+      const stmt = db.prepare(avgQuery)
+      const result = stmt.get()
+
+      return exits.success(result.average)
+    } catch (err) {
+      return exits.error(err)
+    }
+  }
+}

package/lib/private/machines/count-records.js

@@ -0,0 +1,78 @@
+const buildSqliteWhereClause = require('./private/build-sqlite-where-clause')
+
+module.exports = {
+  friendlyName: 'Count (records)',
+
+  description: 'Return the count of the records matched by the query.',
+
+  inputs: {
+    query: require('../constants/query.input'),
+    connection: require('../constants/connection.input'),
+    dryOrm: require('../constants/dry-orm.input')
+  },
+
+  exits: {
+    success: {
+      outputFriendlyName: 'Total (# of records)',
+      outputDescription: 'The number of matching records.',
+      outputExample: 59
+    }
+  },
+
+  fn: function (inputs, exits) {
+    const s3q = inputs.query
+    if (s3q.meta && s3q.meta.logSqliteS3Qs) {
+      console.log(
+        '* * * * * *\nADAPTER (COUNT RECORDS):',
+        require('util').inspect(s3q, { depth: 5 }),
+        '\n'
+      )
+    }
+
+    const tableName = s3q.using
+
+    // Find model by tableName since models is an object, not an array
+    let WLModel = null
+    for (const modelIdentity in inputs.dryOrm.models) {
+      if (inputs.dryOrm.models[modelIdentity].tableName === tableName) {
+        WLModel = inputs.dryOrm.models[modelIdentity]
+        break
+      }
+    }
+    if (!WLModel) {
+      return exits.error(
+        new Error(
+          `No model with that tableName (\`${tableName}\`) has been registered with this adapter. Were any unexpected modifications made to the stage 3 query? Could the adapter's internal state have been corrupted? (This error is usually due to a bug in this adapter's implementation.)`
+        )
+      )
+    }
+
+    // Build a SQLite WHERE clause from the `where` clause.
+    let whereClause
+    try {
+      whereClause = buildSqliteWhereClause(
+        s3q.criteria.where,
+        WLModel,
+        s3q.meta
+      )
+    } catch (e) {
+      return exits.error(e)
+    }
+
+    const db = inputs.connection
+
+    try {
+      let countQuery = `SELECT COUNT(*) as count FROM ${tableName}`
+      if (whereClause) {
+        countQuery += ` WHERE ${whereClause}`
+      }
+
+      const stmt = db.prepare(countQuery)
+      const result = stmt.get()
+
+      return exits.success(result.count)
+    } catch (err) {
+      return exits.error(err)
+    }
+  }
+}

package/lib/private/machines/create-each-record.js

@@ -0,0 +1,163 @@
+const util = require('util')
+const processNativeRecord = require('./private/process-native-record')
+const processNativeError = require('./private/process-native-error')
+const reifyValuesToSet = require('./private/reify-values-to-set')
+
+module.exports = {
+  friendlyName: 'Create each (record)',
+
+  description: 'Insert multiple records into a table in the SQLite database.',
+
+  inputs: {
+    query: require('../constants/query.input'),
+    connection: require('../constants/connection.input'),
+    dryOrm: require('../constants/dry-orm.input')
+  },
+
+  exits: {
+    success: {
+      outputFriendlyName: 'Records (maybe)',
+      outputDescription:
+        'Either `null` or (if `fetch:true`) an array of new physical records that were created.',
+      outputExample: '==='
+    },
+    notUnique: require('../constants/not-unique.exit')
+  },
+
+  fn: async function (inputs, exits) {
+    const s3q = inputs.query
+    if (s3q.meta && s3q.meta.logSQLiteS3Qs) {
+      console.log(
+        '* * * * * *\nADAPTER (CREATE EACH RECORD):',
+        util.inspect(s3q, { depth: 5 }),
+        '\n'
+      )
+    }
+
+    const tableName = s3q.using
+    // Find model by tableName since models is an object, not an array
+    let WLModel = null
+    for (const modelIdentity in inputs.dryOrm.models) {
+      if (inputs.dryOrm.models[modelIdentity].tableName === tableName) {
+        WLModel = inputs.dryOrm.models[modelIdentity]
+        break
+      }
+    }
+
+    if (!WLModel) {
+      return exits.error(
+        new Error(
+          `No model with that tableName (\`${tableName}\`) has been registered with this adapter. Were any unexpected modifications made to the stage 3 query? Could the adapter's internal state have been corrupted? (This error is usually due to a bug in this adapter's implementation.)`
+        )
+      )
+    }
+
+    try {
+      s3q.newRecords.forEach((newRecord) => {
+        reifyValuesToSet(newRecord, WLModel, s3q.meta)
+      })
+    } catch (e) {
+      return exits.error(e)
+    }
+
+    const isFetchEnabled = !!(s3q.meta && s3q.meta.fetch)
+
+    const db = inputs.connection
+
+    // Validate records array
+    if (!Array.isArray(s3q.newRecords) || s3q.newRecords.length === 0) {
+      return exits.error(
+        new Error(
+          'Cannot create records: no data provided or invalid data format'
+        )
+      )
+    }
+
+    try {
+      // Performance optimization: Use a single INSERT statement with multiple VALUES
+      // This is much more efficient than individual INSERT statements
+      const firstRecord = s3q.newRecords[0]
+      const columnNames = Object.keys(firstRecord)
+
+      // Validate that all records have the same columns
+      const invalidRecord = s3q.newRecords.find((record) => {
+        const recordColumns = Object.keys(record)
+        return (
+          recordColumns.length !== columnNames.length ||
+          !recordColumns.every((col) => columnNames.includes(col))
+        )
+      })
+
+      if (invalidRecord) {
+        throw new Error(
+          'All records must have the same columns for batch insert'
+        )
+      }
+
+      const columns = columnNames.join(', ')
+      const valueClause = `(${columnNames.map(() => '?').join(', ')})`
+      const allValueClauses = Array(s3q.newRecords.length)
+        .fill(valueClause)
+        .join(', ')
+      const sql = `INSERT INTO \`${tableName}\` (${columns}) VALUES ${allValueClauses}`
+
+      // Flatten all values for the batch insert
+      const allValues = s3q.newRecords.flatMap((record) =>
+        columnNames.map((col) => record[col])
+      )
+
+      // Use transaction for atomic batch insert - recommended for performance
+      let insertInfo
+      if (db.runInTransaction) {
+        insertInfo = db.runInTransaction(() => {
+          const stmt = db.getPreparedStatement
+            ? db.getPreparedStatement(sql)
+            : db.prepare(sql)
+          return stmt.run(allValues)
+        })
+      } else {
+        // Fallback transaction approach
+        const transaction = db.transaction(() => {
+          const stmt = db.prepare(sql)
+          return stmt.run(allValues)
+        })
+        insertInfo = transaction()
+      }
+
+      // If `fetch` is NOT enabled, we're done.
+      if (!isFetchEnabled) {
+        return exits.success()
+      }
+
+      // For batch inserts, we need to calculate the range of inserted IDs
+      // SQLite auto-increments IDs sequentially in a transaction
+      const lastInsertRowid = insertInfo.lastInsertRowid
+      const recordCount = s3q.newRecords.length
+      const firstInsertRowid = lastInsertRowid - recordCount + 1
+
+      // Fetch the inserted records using the ID range
+      const selectSql = `SELECT * FROM \`${tableName}\` WHERE rowid >= ? AND rowid <= ? ORDER BY rowid`
+      const selectStmt = db.prepare(selectSql)
+      const phRecords = selectStmt.all(firstInsertRowid, lastInsertRowid)
+
+      if (phRecords.length !== recordCount) {
+        throw new Error(
+          `Consistency violation: Expected ${recordCount} records but retrieved ${phRecords.length}`
+        )
+      }
+
+      // Process records in place for better performance
+      phRecords.forEach((phRecord) => {
+        processNativeRecord(phRecord, WLModel, s3q.meta)
+      })
+
+      return exits.success(phRecords)
+    } catch (err) {
+      err = processNativeError(err)
+      if (err.footprint && err.footprint.identity === 'notUnique') {
+        return exits.notUnique(err)
+      }
+      return exits.error(err)
+    }
+  }
+}

package/lib/private/machines/create-manager.js

@@ -0,0 +1,174 @@
+module.exports = {
+  friendlyName: 'Create manager',
+
+  description: 'Build and initialize a connection manager instance for SQLite.',
+
+  inputs: {
+    connectionString: {
+      description: 'The SQLite connection string (file path).',
+      example: 'db/database.sqlite',
+      required: true
+    },
+
+    meta: {
+      friendlyName: 'Meta (custom)',
+      description:
+        'A dictionary of additional options to pass in when instantiating the SQLite client.',
+      example: '==='
+    }
+  },
+
+  exits: {
+    success: {
+      description: 'Connected to SQLite successfully.',
+      outputFriendlyName: 'Report',
+      outputDescription:
+        'The `manager` property is a SQLite database instance.',
+      outputExample: '==='
+    }
+  },
+
+  fn: function ({ connectionString, meta }, exits) {
+    const Database = require('better-sqlite3')
+    const path = require('path')
+    const fs = require('fs')
+
+    try {
+      // Ensure the directory exists for the database file
+      const dbDir = path.dirname(connectionString)
+      if (dbDir !== '.' && !fs.existsSync(dbDir)) {
+        fs.mkdirSync(dbDir, { recursive: true })
+      }
+
+      // Create database connection with optimized options
+      const dbOptions = {
+        // Enable verbose mode in development
+        verbose:
+          meta?.verbose || process.env.NODE_ENV === 'development'
+            ? console.log
+            : null,
+        // Set timeout for database operations
+        timeout: meta?.timeout || 5000,
+        // Enable read-only mode if specified
+        readonly: meta?.readonly || false,
+        // Enable file must exist mode if specified
+        fileMustExist: meta?.fileMustExist || false,
+        ...meta
+      }
+
+      const db = new Database(connectionString, dbOptions)
+
+      // Apply recommended performance pragmas for optimal SQLite performance
+      const defaultPragmas = {
+        // WAL mode for better concurrency (default)
+        journal_mode: 'WAL',
+        // Synchronous mode for better performance vs durability balance
+        synchronous: 'NORMAL',
+        // Enable foreign key support
+        foreign_keys: 'ON',
+        // Set cache size to 256MB (negative value means KB)
+        cache_size: -262144,
+        // Set page size to 4KB (recommended for modern systems)
+        page_size: 4096,
+        // Optimize for read-heavy workloads
+        optimize: true,
+        // Enable memory-mapped I/O
+        mmap_size: 268435456, // 256MB
+        // Set busy timeout to 30 seconds
+        busy_timeout: 30000,
+        // Enable automatic index creation for WHERE clauses
+        automatic_index: 'ON',
+        // Optimize temp store for performance
+        temp_store: 'MEMORY'
+      }
+
+      // Merge with user-provided pragmas
+      const pragmas = { ...defaultPragmas, ...(meta?.pragmas || {}) }
+
+      // Apply pragmas with error handling
+      Object.entries(pragmas).forEach(([key, value]) => {
+        if (value !== false && value !== null && value !== undefined) {
+          try {
+            db.pragma(`${key} = ${value}`)
+          } catch (pragmaError) {
+            console.warn(
+              `Warning: Could not set pragma ${key} = ${value}:`,
+              pragmaError.message
+            )
+          }
+        }
+      })
+
+      // Run ANALYZE to update query planner statistics
+      // This is especially important for new databases
+      try {
+        db.exec('ANALYZE')
+      } catch (analyzeError) {
+        // ANALYZE might fail on empty database, which is fine
+        console.debug(
+          'ANALYZE command failed (this is normal for new databases):',
+          analyzeError.message
+        )
+      }
+
+      // Prepare commonly used statements for better performance
+      // These will be cached and reused throughout the application lifecycle
+      const preparedStatements = new Map()
+
+      // Add helper method to get or create prepared statements
+      db.getPreparedStatement = function (sql) {
+        if (!preparedStatements.has(sql)) {
+          preparedStatements.set(sql, this.prepare(sql))
+        }
+        return preparedStatements.get(sql)
+      }
+
+      // Add transaction helper methods for better performance
+      db.runInTransaction = function (fn) {
+        return this.transaction(fn)()
+      }
+
+      // Add method to optimize database
+      db.optimize = function () {
+        this.exec('PRAGMA optimize')
+        this.exec('VACUUM')
+        this.exec('ANALYZE')
+      }
+
+      // Add graceful cleanup method
+      db.closeGracefully = function () {
+        // Clear prepared statements - newer better-sqlite3 doesn't need explicit finalize
+        preparedStatements.clear()
+
+        // Close the database connection
+        if (this.open) {
+          this.close()
+        }
+      }
+
+      // Set up connection health check
+      db.isHealthy = function () {
+        try {
+          this.prepare('SELECT 1').get()
+          return true
+        } catch (error) {
+          return false
+        }
+      }
+
+      return exits.success({
+        manager: db,
+        meta: {
+          ...meta,
+          connectionString,
+          pragmasApplied: pragmas,
+          connectionEstablishedAt: new Date().toISOString()
+        }
+      })
+    } catch (error) {
+      return exits.error(
+        new Error(`Failed to create SQLite database manager: ${error.message}`)
+      )
+    }
+  }
+}