plugin-data-source-mssql 2.0.11 → 2.0.12

package/src/server/__tests__/mssql-selected-collections.test.ts

@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { MssqlExternalDataSource, type MssqlDataSourceOptions } from '../data-source/MssqlExternalDataSource';
+import type { Context } from '@nocobase/actions';
+import { describe, expect, it, vi } from 'vitest';
+
+function createDataSource(options: MssqlDataSourceOptions) {
+  const getTables = vi.fn().mockResolvedValue(['users', 'orders', 'products']);
+  const preloadAllColumns = vi.fn().mockResolvedValue(undefined);
+  const preloadAllPrimaryKeys = vi.fn().mockResolvedValue(undefined);
+  const preloadFTSIndexedColumns = vi.fn().mockResolvedValue(undefined);
+  const getCollection = vi.fn(async ({ tableInfo, localOptions }) => ({
+    name: tableInfo.tableName,
+    title: localOptions.title ?? tableInfo.tableName,
+    fields: localOptions.fields ?? [{ name: 'id', type: 'integer' }],
+  }));
+  const defineCollection = vi.fn();
+  const authenticate = vi.fn().mockResolvedValue(undefined);
+  const registerOperators = vi.fn();
+  const db = {
+    options: { schema: 'dbo' },
+    sequelize: { authenticate },
+    checkVersion: vi.fn().mockResolvedValue(undefined),
+    registerOperators,
+  };
+  const introspector = {
+    getTables,
+    tableInfoToCollectionOptions: ({ tableName }: { tableName: string }) => ({ name: tableName }),
+    getCollection,
+    preloadAllColumns,
+    preloadAllPrimaryKeys,
+    preloadFTSIndexedColumns,
+  };
+  const dataSource = Object.create(MssqlExternalDataSource.prototype) as MssqlExternalDataSource;
+  Object.assign(dataSource, {
+    options,
+    collectionManager: { db, defineCollection },
+    introspector,
+    logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
+    emitLoadingProgress: vi.fn(),
+  });
+
+  return {
+    dataSource,
+    getTables,
+    getCollection,
+    preloadAllColumns,
+    preloadAllPrimaryKeys,
+    preloadFTSIndexedColumns,
+    defineCollection,
+    registerOperators,
+    authenticate,
+  };
+}
+
+describe('MSSQL selected collection loading', () => {
+  it('introspects only saved selections and merges each with its persisted local metadata', async () => {
+    const localCollection = {
+      name: 'orders',
+      title: 'Sales orders',
+      fields: [{ name: 'orderTotal', type: 'decimal', title: 'Total' }],
+    };
+    const spies = createDataSource({
+      addAllCollections: false,
+      selectedCollections: ['orders'],
+    });
+
+    await spies.dataSource.load({ localData: { orders: localCollection } });
+
+    expect(spies.authenticate).toHaveBeenCalledOnce();
+    expect(spies.getTables).not.toHaveBeenCalled();
+    expect(spies.preloadAllColumns).not.toHaveBeenCalled();
+    expect(spies.preloadAllPrimaryKeys).not.toHaveBeenCalled();
+    expect(spies.preloadFTSIndexedColumns).toHaveBeenCalledWith('dbo', ['orders']);
+    expect(spies.getCollection).toHaveBeenCalledTimes(1);
+    expect(spies.getCollection).toHaveBeenCalledWith({
+      tableInfo: { tableName: 'orders', schema: 'dbo' },
+      localOptions: localCollection,
+    });
+    expect(spies.defineCollection).toHaveBeenCalledWith(
+      expect.objectContaining({
+        name: 'orders',
+        title: 'Sales orders',
+        fields: localCollection.fields,
+        introspected: true,
+        repository: 'mssql-repo',
+      }),
+    );
+  });
+
+  it('does no external schema introspection when the saved selection is empty', async () => {
+    const spies = createDataSource({ addAllCollections: false, selectedCollections: [] });
+
+    await spies.dataSource.load();
+
+    expect(spies.authenticate).toHaveBeenCalledOnce();
+    expect(spies.getTables).not.toHaveBeenCalled();
+    expect(spies.preloadAllColumns).not.toHaveBeenCalled();
+    expect(spies.preloadAllPrimaryKeys).not.toHaveBeenCalled();
+    expect(spies.preloadFTSIndexedColumns).not.toHaveBeenCalled();
+    expect(spies.getCollection).not.toHaveBeenCalled();
+    expect(spies.defineCollection).not.toHaveBeenCalled();
+  });
+
+  it.each(['create', 'update'] as const)(
+    'persists the selected collection names on datasource %s options',
+    async (actionName) => {
+      const spies = createDataSource({ addAllCollections: false });
+      const ctx = {
+        action: {
+          actionName,
+          resourceName: 'dataSources',
+          params: {
+            values: {
+              options: { host: 'db.test', addAllCollections: false },
+              collections: ['orders', 'users', 'orders'],
+            },
+          },
+        },
+      } as unknown as Context;
+
+      await spies.dataSource.loadTables(ctx, ['orders', 'users', 'orders']);
+
+      expect(ctx.action.params.values.options).toMatchObject({
+        host: 'db.test',
+        addAllCollections: false,
+        selectedCollections: ['orders', 'users'],
+      });
+    },
+  );
+
+  it('persists an empty selection rather than treating it as a missing selection', async () => {
+    const spies = createDataSource({ addAllCollections: false });
+    const ctx = {
+      action: {
+        actionName: 'create',
+        resourceName: 'dataSources',
+        params: {
+          values: { options: { addAllCollections: false }, collections: [] },
+        },
+      },
+    } as unknown as Context;
+
+    await spies.dataSource.loadTables(ctx, []);
+
+    expect(ctx.action.params.values.options).toMatchObject({ addAllCollections: false, selectedCollections: [] });
+  });
+
+  it.each([
+    ['explicit load all', { addAllCollections: true }],
+    ['legacy false without a saved selection', { addAllCollections: false }],
+    ['legacy missing load-all option', {}],
+  ])('%s retains the historical full-load behavior', async (_label, options) => {
+    const spies = createDataSource(options);
+
+    await spies.dataSource.load();
+
+    expect(spies.getTables).toHaveBeenCalledOnce();
+    expect(spies.preloadAllColumns).toHaveBeenCalledWith('dbo');
+    expect(spies.preloadAllPrimaryKeys).toHaveBeenCalledWith('dbo');
+    expect(spies.preloadFTSIndexedColumns).toHaveBeenCalledWith('dbo', undefined);
+    expect(spies.getCollection).toHaveBeenCalledTimes(3);
+    expect(spies.defineCollection).toHaveBeenCalledTimes(3);
+  });
+});

package/src/server/__tests__/mssql-smart-cursor-proof.test.ts

@@ -0,0 +1,64 @@
+import { MssqlSmartCursorBuilder } from '../data-source/MssqlSmartCursorBuilder';
+
+function makeSequelize(indexRows: any[]) {
+  return { getDialect: () => 'mssql', query: async () => indexRows } as any;
+}
+const collection = { filterTargetKey: 'id' } as any;
+
+// depth of Op.and nesting
+function andDepth(where: any): number {
+  const syms = Object.getOwnPropertySymbols(where || {});
+  const andSym = syms.find((s) => s.toString() === 'Symbol(and)');
+  if (!andSym) return 0;
+  const arr = where[andSym] || [];
+  return 1 + Math.max(0, ...arr.map(andDepth));
+}
+
+describe('where accumulation proof', () => {
+  it('single-column cursor does NOT accumulate (shallow merge overwrite)', async () => {
+    const sequelize = makeSequelize([
+      { INDEX_NAME: 'pk', COLUMN_NAME: 'id', SEQ_IN_INDEX: 1, INDEX_TYPE: 1, DIRECTION: 'ASC' },
+    ]);
+    const builder = new MssqlSmartCursorBuilder(sequelize, 'T', collection);
+    const depths: number[] = [];
+    let batch = 0;
+    await builder.chunk({
+      chunkSize: 2,
+      where: { status: 'active' },
+      callback: async () => {},
+      find: async (opts: any) => {
+        depths.push(andDepth(opts.where));
+        batch++;
+        return batch <= 3 ? [{ id: batch }, { id: batch + 10 }] : [];
+      },
+    } as any);
+    expect(depths).toEqual([0, 0, 0, 0]); // never wraps in Op.and
+  });
+
+  it('composite-key cursor NO ACCUMULATION after fix: Op.and nesting stays constant', async () => {
+    const sequelize = makeSequelize([
+      { INDEX_NAME: 'pk', COLUMN_NAME: 'a', SEQ_IN_INDEX: 1, INDEX_TYPE: 1, DIRECTION: 'ASC' },
+      { INDEX_NAME: 'pk', COLUMN_NAME: 'b', SEQ_IN_INDEX: 2, INDEX_TYPE: 1, DIRECTION: 'ASC' },
+    ]);
+    const builder = new MssqlSmartCursorBuilder(sequelize, 'T', collection);
+    const depths: number[] = [];
+    let batch = 0;
+    await builder.chunk({
+      chunkSize: 2,
+      where: { status: 'active' },
+      callback: async () => {},
+      find: async (opts: any) => {
+        depths.push(andDepth(opts.where));
+        batch++;
+        return batch <= 3
+          ? [
+              { a: batch, b: batch },
+              { a: batch + 10, b: batch + 10 },
+            ]
+          : [];
+      },
+    } as any);
+    // batch1: no Op.and (just base where). Subsequent batches: exactly 1 Op.and level, constant across batches.
+    expect(depths).toEqual([0, 1, 1, 1]); // FIXED: no accumulation, constant depth
+  });
+});

package/src/server/__tests__/mssql-smart-cursor.test.ts

@@ -0,0 +1,107 @@
+import { MssqlSmartCursorBuilder } from '../data-source/MssqlSmartCursorBuilder';
+import { Op } from 'sequelize';
+
+// Minimal sequelize stub capturing the SQL + replacements actually issued.
+function makeSequelize(indexRows: any[]) {
+  const calls: any[] = [];
+  return {
+    calls,
+    getDialect: () => 'mssql',
+    query: async (sql: string, opts: any) => {
+      calls.push({ sql, replacements: opts.replacements });
+      return indexRows;
+    },
+  } as any;
+}
+
+const collection = { filterTargetKey: 'id' } as any;
+
+describe('MssqlSmartCursorBuilder', () => {
+  describe('getBestCursorStrategy index SQL', () => {
+    it('passes the bracket-quoted table name (no schema) to OBJECT_ID', async () => {
+      const sequelize = makeSequelize([]);
+      const builder = new MssqlSmartCursorBuilder(sequelize, 'Users', collection);
+      await builder.chunk({ chunkSize: 10, callback: async () => {}, find: async () => [] } as any);
+      expect(sequelize.calls[0].replacements).toEqual(['[Users]']);
+    });
+
+    it('passes schema-qualified bracket-quoted name to OBJECT_ID when schema is given', async () => {
+      const sequelize = makeSequelize([]);
+      const builder = new MssqlSmartCursorBuilder(sequelize, 'Clients', collection, 'identity');
+      await builder.chunk({ chunkSize: 10, callback: async () => {}, find: async () => [] } as any);
+      expect(sequelize.calls[0].replacements).toEqual(['[identity].[Clients]']);
+    });
+
+    it('escapes a closing bracket in table or schema name', async () => {
+      const sequelize = makeSequelize([]);
+      const builder = new MssqlSmartCursorBuilder(sequelize, 'we]ird', collection, 'sch]ema');
+      await builder.chunk({ chunkSize: 10, callback: async () => {}, find: async () => [] } as any);
+      expect(sequelize.calls[0].replacements).toEqual(['[sch]]ema].[we]]ird]']);
+    });
+    it('ORDER BY index name (alphabetical), not by primary/unique priority', async () => {
+      const sequelize = makeSequelize([]);
+      const builder = new MssqlSmartCursorBuilder(sequelize, 'Users', collection);
+      await builder.chunk({ chunkSize: 10, callback: async () => {}, find: async () => [] } as any);
+      expect(sequelize.calls[0].sql).toMatch(/ORDER BY\s+i\.name/i);
+    });
+
+    it('still picks the PRIMARY KEY index even when a non-PK index sorts first alphabetically', async () => {
+      // aaa_idx (non-unique, column nonUniqueCol) sorts before pk_users (PK, column id)
+      const rows = [
+        { INDEX_NAME: 'aaa_idx', COLUMN_NAME: 'nonUniqueCol', SEQ_IN_INDEX: 1, INDEX_TYPE: 3, DIRECTION: 'ASC' },
+        { INDEX_NAME: 'pk_users', COLUMN_NAME: 'id', SEQ_IN_INDEX: 1, INDEX_TYPE: 1, DIRECTION: 'ASC' },
+      ];
+      const sequelize = makeSequelize(rows);
+      const builder = new MssqlSmartCursorBuilder(sequelize, 'Users', collection);
+      const strategy = await (builder as any).getBestCursorStrategy();
+      expect(strategy.columnName).toBe('id');
+    });
+
+    it('orders composite index columns via sparse key_ordinal assignment', async () => {
+      // Composite PK out of row order: key_ordinal 2 arrives before 1
+      const rows = [
+        { INDEX_NAME: 'pk_c', COLUMN_NAME: 'b', SEQ_IN_INDEX: 2, INDEX_TYPE: 1, DIRECTION: 'ASC' },
+        { INDEX_NAME: 'pk_c', COLUMN_NAME: 'a', SEQ_IN_INDEX: 1, INDEX_TYPE: 1, DIRECTION: 'ASC' },
+      ];
+      const sequelize = makeSequelize(rows);
+      const builder = new MssqlSmartCursorBuilder(sequelize, 'Users', collection);
+      const strategy = await (builder as any).getBestCursorStrategy();
+      expect(strategy.buildSort()).toEqual([
+        ['a', 'ASC'],
+        ['b', 'ASC'],
+      ]);
+    });
+  });
+
+  describe('chunk() where accumulation across batches', () => {
+    // 3 batches: batch1 full, batch2 full, batch3 empty -> where wrapped twice
+    it('nests the user where deeper on every subsequent batch', async () => {
+      const sequelize = makeSequelize([
+        { INDEX_NAME: 'pk_users', COLUMN_NAME: 'id', SEQ_IN_INDEX: 1, INDEX_TYPE: 1, DIRECTION: 'ASC' },
+      ]);
+      const builder = new MssqlSmartCursorBuilder(sequelize, 'Users', collection);
+
+      const seenWheres: any[] = [];
+      let batch = 0;
+      await builder.chunk({
+        chunkSize: 2,
+        where: { status: 'active' },
+        callback: async () => {},
+        find: async (opts: any) => {
+          seenWheres.push(
+            JSON.parse(JSON.stringify(opts.where ?? {}, (k, v) => (typeof v === 'symbol' ? v.toString() : v))),
+          );
+          batch++;
+          if (batch === 1) return [{ id: 1 }, { id: 2 }];
+          if (batch === 2) return [{ id: 3 }, { id: 4 }];
+          return [];
+        },
+      } as any);
+
+      // batch1: plain user where
+      expect(seenWheres[0]).toEqual({ status: 'active' });
+      // batch2: user where + id>2 merged at top level (SingleColumn strategy shallow-merges)
+      expect(seenWheres[1]).toMatchObject({ status: 'active' });
+    });
+  });
+});

package/src/server/__tests__/sql-quote.test.ts

@@ -0,0 +1,103 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+import {
+  isSafeUnquotedIdentifier,
+  quoteIdentifier,
+  quoteQualifiedIdentifier,
+  quoteUnicodeStringLiteral,
+  formatDateTimeParameter,
+} from '../security/sql-quote';
+
+describe('sql-quote security helpers', () => {
+  describe('isSafeUnquotedIdentifier', () => {
+    it('accepts plain identifiers', () => {
+      expect(isSafeUnquotedIdentifier('username')).toBe(true);
+      expect(isSafeUnquotedIdentifier('Order_Items2')).toBe(true);
+      expect(isSafeUnquotedIdentifier('_id')).toBe(true);
+    });
+
+    it('rejects identifiers with special chars', () => {
+      expect(isSafeUnquotedIdentifier('user name')).toBe(false);
+      expect(isSafeUnquotedIdentifier('a-b')).toBe(false);
+      expect(isSafeUnquotedIdentifier('a.b')).toBe(false);
+      expect(isSafeUnquotedIdentifier('9lives')).toBe(false);
+      expect(isSafeUnquotedIdentifier('a]b')).toBe(false);
+    });
+
+    it('rejects reserved keywords regardless of case', () => {
+      expect(isSafeUnquotedIdentifier('select')).toBe(false);
+      expect(isSafeUnquotedIdentifier('Order')).toBe(false);
+      expect(isSafeUnquotedIdentifier('CONTAINS')).toBe(false);
+    });
+  });
+
+  describe('quoteIdentifier', () => {
+    it('bracket-quotes a simple identifier when policy is always-quote', () => {
+      expect(quoteIdentifier('username')).toBe('[username]');
+    });
+
+    it('doubles an embedded closing bracket so it cannot terminate the identifier', () => {
+      expect(quoteIdentifier('a]b')).toBe('[a]]b]');
+      expect(quoteIdentifier('x], [injected')).toBe('[x]], [injected]');
+    });
+
+    it('quote-when-needed leaves safe identifiers bare but quotes unsafe ones', () => {
+      expect(quoteIdentifier('username', 'quote-when-needed')).toBe('username');
+      expect(quoteIdentifier('user name', 'quote-when-needed')).toBe('[user name]');
+      expect(quoteIdentifier('Order', 'quote-when-needed')).toBe('[Order]');
+    });
+  });
+
+  describe('quoteQualifiedIdentifier', () => {
+    it('quotes and joins a dotted path', () => {
+      expect(quoteQualifiedIdentifier(['dbo', 'Orders'])).toBe('[dbo].[Orders]');
+    });
+
+    it('escapes brackets in each part', () => {
+      expect(quoteQualifiedIdentifier(['we]ird', 'ta]ble'])).toBe('[we]]ird].[ta]]ble]');
+    });
+  });
+
+  describe('quoteUnicodeStringLiteral', () => {
+    it('prefixes with N and single-quotes', () => {
+      expect(quoteUnicodeStringLiteral('foo')).toBe("N'foo'");
+    });
+
+    it('doubles single quotes to prevent break-out', () => {
+      expect(quoteUnicodeStringLiteral("o'brien")).toBe("N'o''brien'");
+      expect(quoteUnicodeStringLiteral("'; DROP TABLE users; --")).toBe("N'''; DROP TABLE users; --'");
+    });
+
+    it('preserves non-ASCII characters verbatim', () => {
+      expect(quoteUnicodeStringLiteral('tiếng việt')).toBe("N'tiếng việt'");
+    });
+  });
+
+  describe('formatDateTimeParameter', () => {
+    it('formats a plain Date as YYYY-MM-DD HH:mm:ss.SSS (space separator, no Z)', () => {
+      const d = new Date(Date.UTC(2024, 0, 2, 3, 4, 5, 678));
+      expect(formatDateTimeParameter(d)).toBe('2024-01-02 03:04:05.678');
+    });
+
+    it('recovers sub-millisecond precision from tedious nanosecondsDelta', () => {
+      const d = new Date(Date.UTC(2024, 0, 2, 3, 4, 5, 678)) as Date & { nanosecondsDelta?: number };
+      d.nanosecondsDelta = 0.5; // half a millisecond -> 5000 extra 100ns ticks
+      const text = formatDateTimeParameter(d);
+      expect(text.startsWith('2024-01-02 03:04:05.678')).toBe(true);
+      expect(text.length).toBeGreaterThan('2024-01-02 03:04:05.678'.length);
+    });
+
+    it('ignores a zero/negative nanosecondsDelta', () => {
+      const d = new Date(Date.UTC(2024, 0, 2, 3, 4, 5, 678)) as Date & { nanosecondsDelta?: number };
+      d.nanosecondsDelta = 0;
+      expect(formatDateTimeParameter(d)).toBe('2024-01-02 03:04:05.678');
+    });
+  });
+});