windmill-components 1.623.1 → 1.642.0

package/package/components/apps/components/display/dbtable/metadata.js

@@ -2,21 +2,15 @@ import { JobService, ResourceService } from '../../../../../gen';
 import { runScriptAndPollResult } from '../../../../jobs/utils';
 import { getLanguageByResourceType, resourceTypeToLang, legacyScripts, scriptsV2 } from './utils';
 import {} from '../../../../../gen';
-import { tryEvery } from '../../../../../utils';
 import { stringifySchema } from '../../../../copilot/lib';
 import { getDatabaseArg } from '../../../../dbOps';
 export async function loadTableMetaData(input, workspace, table) {
     if (!input || !table || !workspace)
         return undefined;
-    let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType);
-    let content = await makeLoadTableMetaDataQuery(input, workspace, table);
+    let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, table);
     const job = await JobService.runScriptPreview({
         workspace: workspace,
-        requestBody: {
-            language,
-            content,
-            args: getDatabaseArg(input)
-        }
+        requestBody: { language, content: query, args: getDatabaseArg(input) }
     });
     const maxRetries = 8;
     let attempts = 0;
@@ -29,12 +23,15 @@ export async function loadTableMetaData(input, workspace, table) {
             }));
             if (testResult.success) {
                 attempts = maxRetries;
-                if (input.type === 'database' && input.resourceType === 'ms_sql_server') {
-                    return testResult.result[0].map(lowercaseKeys);
-                }
-                else {
-                    return testResult.result.map(lowercaseKeys);
+                const result = testResult.result.map(lowercaseKeys);
+                // For Snowflake, fetch primary keys separately
+                if (input.type === 'database' &&
+                    (input.resourceType === 'snowflake' || input.resourceType === 'snowflake_oauth')) {
+                    const map = { [table]: result };
+                    await fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace, table);
+                    return map[table];
                 }
+                return result;
             }
             else {
                 attempts++;
@@ -50,28 +47,20 @@ export async function loadTableMetaData(input, workspace, table) {
 export async function loadAllTablesMetaData(workspace, input) {
     if (!input || !workspace)
         return undefined;
-    let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType);
     try {
+        let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, undefined);
         let result = (await runScriptAndPollResult({
             workspace: workspace,
-            requestBody: {
-                language,
-                content: await makeLoadTableMetaDataQuery(input, workspace, undefined),
-                args: getDatabaseArg(input)
-            }
+            requestBody: { language, content: query, args: getDatabaseArg(input) }
         }));
-        if (input.type === 'database' && input.resourceType === 'ms_sql_server') {
-            result = result[0];
-        }
         const map = {};
         for (const _col of result) {
             const col = lowercaseKeys(_col);
             const tableKey = col.schema_name ? `${col.schema_name}.${col.table_name}` : col.table_name;
-            if (!(tableKey in map)) {
-                map[tableKey] = [];
-            }
+            map[tableKey] ??= [];
             map[tableKey].push(col);
         }
+        await fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace);
         return map;
     }
     catch (e) {
@@ -80,7 +69,7 @@ export async function loadAllTablesMetaData(workspace, input) {
 }
 async function makeLoadTableMetaDataQuery(input, workspace, table) {
     if (input.type === 'ducklake') {
-        return `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__;
+        const query = `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__;
 		SELECT
 			COLUMN_NAME as field,
 			DATA_TYPE as DataType,
@@ -92,18 +81,19 @@ async function makeLoadTableMetaDataQuery(input, workspace, table) {
 			TABLE_NAME as table_name
 		FROM information_schema.columns c
 		WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()`;
+        return { query, language: 'duckdb' };
     }
     else if (input.resourceType === 'mysql') {
         const resourceObj = (await ResourceService.getResourceValue({
             workspace,
             path: input.resourcePath
         }));
-        return `
+        const query = `
 	SELECT 
 			COLUMN_NAME as field,
 			COLUMN_TYPE as DataType,
 			COLUMN_DEFAULT as DefaultValue,
-			CASE WHEN COLUMN_KEY = 'PRI' THEN 'YES' ELSE 'NO' END as IsPrimaryKey,
+			CASE WHEN COLUMN_KEY = 'PRI' THEN 1 ELSE 0 END as IsPrimaryKey,
 			CASE WHEN EXTRA like '%auto_increment%' THEN 'YES' ELSE 'NO' END as IsIdentity,
 			CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
 			CASE WHEN DATA_TYPE = 'enum' THEN true ELSE false END as IsEnum${table
@@ -122,9 +112,10 @@ async function makeLoadTableMetaDataQuery(input, workspace, table) {
 			TABLE_NAME,
 			ORDINAL_POSITION;
 	`;
+        return { query, language: 'mysql' };
     }
     else if (input.resourceType === 'postgresql') {
-        return `
+        const query = `
 	SELECT 
 		a.attname as field,
 		pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType,
@@ -164,38 +155,59 @@ async function makeLoadTableMetaDataQuery(input, workspace, table) {
 	ORDER BY ${table ? 'a.attnum' : 'ns.nspname, c.relname, a.attnum'};
 	
 	`;
+        return { query, language: 'postgresql' };
     }
     else if (input.resourceType === 'ms_sql_server') {
-        return `
-		SELECT 
-    COLUMN_NAME as field,
-    DATA_TYPE as DataType,
-    COLUMN_DEFAULT as DefaultValue,
-    CASE WHEN COLUMNPROPERTY(OBJECT_ID(TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1 THEN 'By Default' ELSE 'No' END as IsIdentity,
-    CASE WHEN COLUMNPROPERTY(OBJECT_ID(TABLE_NAME), COLUMN_NAME, 'IsIdentity') = 1 THEN 1 ELSE 0 END as IsPrimaryKey, -- This line still needs correction for primary key identification
-    CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
-    CASE WHEN DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum${table
+        const query = `
+		SELECT
+    c.COLUMN_NAME as field,
+    c.DATA_TYPE as DataType,
+    c.COLUMN_DEFAULT as DefaultValue,
+    CASE WHEN COLUMNPROPERTY(OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME), c.COLUMN_NAME, 'IsIdentity') = 1 THEN 'By Default' ELSE 'No' END as IsIdentity,
+    CASE WHEN pk.COLUMN_NAME IS NOT NULL THEN 1 ELSE 0 END as IsPrimaryKey,
+    CASE WHEN c.IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
+    CASE WHEN c.DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum,
+    dc.name as default_constraint_name${table
             ? ''
             : `,
-		TABLE_NAME as table_name`}
-FROM    
-    INFORMATION_SCHEMA.COLUMNS${table
+		c.TABLE_NAME as table_name`}
+FROM
+    INFORMATION_SCHEMA.COLUMNS c
+    LEFT JOIN (
+        SELECT
+            ku.TABLE_SCHEMA,
+            ku.TABLE_NAME,
+            ku.COLUMN_NAME
+        FROM
+            INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc
+            INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE ku
+                ON tc.CONSTRAINT_TYPE = 'PRIMARY KEY'
+                AND tc.CONSTRAINT_NAME = ku.CONSTRAINT_NAME
+                AND tc.TABLE_SCHEMA = ku.TABLE_SCHEMA
+                AND tc.TABLE_NAME = ku.TABLE_NAME
+    ) pk ON c.TABLE_SCHEMA = pk.TABLE_SCHEMA
+        AND c.TABLE_NAME = pk.TABLE_NAME
+        AND c.COLUMN_NAME = pk.COLUMN_NAME
+    LEFT JOIN sys.default_constraints dc
+        ON dc.parent_object_id = OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME)
+        AND dc.parent_column_id = COLUMNPROPERTY(OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME), c.COLUMN_NAME, 'ColumnId')${table
             ? `
-WHERE   
-    TABLE_NAME = '${table}'`
+WHERE
+    c.TABLE_NAME = '${table}'`
             : ''}
 ORDER BY
-    ORDINAL_POSITION;
+    c.ORDINAL_POSITION;
 	`;
+        return { query, language: 'mssql' };
     }
     else if (input.resourceType === 'snowflake' ||
         input.resourceType === 'snowflake_oauth') {
-        return `
+        const query = `
 		select COLUMN_NAME as field,
 		DATA_TYPE as DataType,
 		COLUMN_DEFAULT as DefaultValue,
 		CASE WHEN COLUMN_DEFAULT like 'AUTOINCREMENT%' THEN 'By Default' ELSE 'No' END as IsIdentity,
-		CASE WHEN COLUMN_DEFAULT like 'AUTOINCREMENT%' THEN 1 ELSE 0 END as IsPrimaryKey,
+		0 as IsPrimaryKey, -- a one-query solution is not trivial, we will use SHOW PRIMARY KEYS separately
 		CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
 		CASE WHEN DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum${table
             ? ''
@@ -208,12 +220,11 @@ ORDER BY
             : "\nwhere table_schema <> 'INFORMATION_SCHEMA'\n"}
 	order by ORDINAL_POSITION;
 	`;
+        return { query, language: 'snowflake' };
     }
     else if (input.resourceType === 'bigquery') {
-        // TODO: find a solution for this (query uses hardcoded dataset name)
-        if (!table)
-            throw new Error('Table name is required for BigQuery');
-        return `SELECT 
+        if (table) {
+            const query = `SELECT 
     c.COLUMN_NAME as field,
     DATA_TYPE as DataType,
     CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue,
@@ -229,11 +240,77 @@ FROM
 WHERE   
     c.TABLE_NAME = '${table.split('.')[1]}'
 order by c.ORDINAL_POSITION;`;
+            return { query, language: 'bigquery' };
+        }
+        else {
+            const query = `import { BigQuery } from '@google-cloud/bigquery@7.5.0';
+export async function main(database: bigquery) {
+const bq = new BigQuery({
+	credentials: database
+})
+const [datasets] = await bq.getDatasets();
+if (!datasets) return {}
+const schema = {} as any
+let queries = datasets.map(dataset => \`
+	(SELECT 
+    c.COLUMN_NAME as field,
+		'\${dataset.id}' as schema_name,
+		c.TABLE_NAME as table_name,
+    DATA_TYPE as DataType,
+    CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue,
+    CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey,
+    'No' as IsIdentity,
+    IS_NULLABLE as IsNullable,
+    false as IsEnum
+FROM
+    \\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.COLUMNS c
+    LEFT JOIN
+    \\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p
+    on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME
+ORDER BY c.ORDINAL_POSITION)\`
+)
+let query = queries.join('\\nUNION ALL \\n')
+const [rows] = await bq.query(query)
+return rows
+}`;
+            return { query, language: 'bun' };
+        }
     }
     else {
         throw new Error('Unsupported database type:' + input.resourceType);
     }
 }
+// We can't get primary keys in a single query for Snowflake, so we fetch them separately
+async function fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace, tableKey) {
+    if (input.type == 'database' &&
+        (input.resourceType === 'snowflake' || input.resourceType === 'snowflake_oauth')) {
+        let pkResult = await fetchSnowflakePrimaryKeys(workspace, getDatabaseArg(input), tableKey);
+        for (const pk of pkResult) {
+            const pkTableKey = `${pk.schema_name}.${pk.table_name}`.toUpperCase();
+            // Also check the original casing and the provided tableKey
+            const keysToCheck = [pkTableKey, `${pk.schema_name}.${pk.table_name}`, tableKey].filter(Boolean);
+            for (const key of keysToCheck) {
+                if (key in map) {
+                    for (const col of map[key]) {
+                        if (col.field.toLowerCase() === pk.column_name.toLowerCase()) {
+                            col.isprimarykey = true;
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+async function fetchSnowflakePrimaryKeys(workspace, dbArg, tableKey) {
+    return (await JobService.runScriptPreviewAndWaitResult({
+        workspace,
+        requestBody: {
+            language: 'snowflake',
+            args: dbArg,
+            content: tableKey ? `SHOW PRIMARY KEYS IN TABLE ${tableKey}` : 'SHOW PRIMARY KEYS IN ACCOUNT'
+        }
+    }));
+}
 function lowercaseKeys(obj) {
     const newObj = {};
     Object.keys(obj).forEach((key) => {
@@ -241,113 +318,66 @@ function lowercaseKeys(obj) {
     });
     return newObj;
 }
-export async function getDbSchemas(resourceType, resourcePath, workspace, dbSchemas, errorCallback, options = {}) {
+export async function getDbSchemas(resourceType, resourcePath, workspace, errorCallback, options = {}) {
     let scripts = options.useLegacyScripts ? legacyScripts : scriptsV2;
-    if (!scripts[resourceType])
+    let sqlScript = scripts[getLanguageByResourceType(resourceType)];
+    if (!resourceType || !resourcePath || !workspace || !sqlScript)
         return;
-    return new Promise(async (resolve, reject) => {
-        if (!resourceType || !resourcePath || !workspace) {
-            resolve();
-            return;
-        }
-        const job = await JobService.runScriptPreview({
-            workspace: workspace,
+    let result;
+    try {
+        result = await JobService.runScriptPreviewAndWaitResult({
+            workspace,
             requestBody: {
-                language: scripts[resourceType].lang,
-                content: scripts[resourceType].code,
+                language: sqlScript.lang,
+                content: sqlScript.code,
+                tag: options.customTag,
                 args: {
-                    [scripts[resourceType].argName]: resourcePath.startsWith('datatable://')
+                    [sqlScript.argName]: resourcePath.startsWith('datatable://')
                         ? resourcePath
                         : '$res:' + resourcePath
                 }
             }
         });
-        tryEvery({
-            tryCode: async () => {
-                if (resourcePath) {
-                    const testResult = await JobService.getCompletedJob({
-                        workspace,
-                        id: job
-                    });
-                    if (!testResult.success) {
-                        console.error(testResult.result?.['error']?.['message']);
-                    }
-                    else {
-                        if (testResult.result === 'WINDMILL_TOO_BIG') {
-                            console.info('Result is too big, fetching result separately');
-                            const data = await JobService.getCompletedJobResult({
-                                workspace,
-                                id: job
-                            });
-                            testResult.result = data;
-                        }
-                        if (resourceType !== undefined) {
-                            if (resourceType !== 'graphql') {
-                                const { processingFn } = scripts[resourceType];
-                                let schema;
-                                try {
-                                    schema =
-                                        processingFn !== undefined ? processingFn(testResult.result) : testResult.result;
-                                }
-                                catch (e) {
-                                    console.error(e);
-                                    errorCallback('Error processing schema');
-                                    resolve();
-                                    return;
-                                }
-                                const dbSchema = {
-                                    lang: resourceTypeToLang(resourceType),
-                                    schema,
-                                    publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
-                                };
-                                dbSchemas[resourcePath] = {
-                                    ...dbSchema,
-                                    stringified: stringifySchema(dbSchema)
-                                };
-                            }
-                            else {
-                                if (typeof testResult.result !== 'object' ||
-                                    !('__schema' in (testResult?.result ?? {}))) {
-                                    console.error('Invalid GraphQL schema');
-                                    errorCallback('Invalid GraphQL schema');
-                                }
-                                else {
-                                    const dbSchema = {
-                                        lang: 'graphql',
-                                        schema: testResult.result
-                                    };
-                                    dbSchemas[resourcePath] = {
-                                        ...dbSchema,
-                                        stringified: stringifySchema(dbSchema)
-                                    };
-                                }
-                            }
-                        }
-                    }
-                    resolve();
-                }
-            },
-            timeoutCode: async () => {
-                console.error('Could not query schema within 5s');
-                errorCallback('Could not query schema within 5s');
-                try {
-                    await JobService.cancelQueuedJob({
-                        workspace,
-                        id: job,
-                        requestBody: {
-                            reason: 'Could not query schema within 5s'
-                        }
-                    });
-                }
-                catch (err) {
-                    console.error(err);
-                }
-                reject();
-            },
-            interval: 500,
-            timeout: 5000
-        });
-    });
+    }
+    catch (e) {
+        console.error(e);
+        errorCallback('Error fetching schema: ' + (e?.message || e));
+        return;
+    }
+    if (resourceType !== undefined) {
+        if (resourceType !== 'graphql') {
+            const { processingFn } = sqlScript;
+            let schema;
+            try {
+                schema = processingFn !== undefined ? processingFn(result) : result;
+            }
+            catch (e) {
+                console.error(e);
+                errorCallback('Error processing schema');
+                return;
+            }
+            const dbSchema = {
+                lang: resourceTypeToLang(resourceType),
+                schema,
+                publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
+            };
+            return { ...dbSchema, stringified: stringifySchema(dbSchema) };
+        }
+        else {
+            if (typeof result !== 'object' || !('__schema' in (result ?? {}))) {
+                console.error('Invalid GraphQL schema');
+                errorCallback('Invalid GraphQL schema');
+                return;
+            }
+            else {
+                const dbSchema = {
+                    lang: 'graphql',
+                    schema: result
+                };
+                return { ...dbSchema, stringified: stringifySchema(dbSchema) };
+            }
+        }
+    }
 }
 export async function getTablesByResource(schema, dbType, dbPath, workspace) {
     const s = Object.values(schema)?.[0];

package/package/components/apps/components/display/dbtable/queries/alterTable.d.ts

@@ -0,0 +1,46 @@
+import type { DbType } from '../../../../../dbTypes';
+import { type TableEditorForeignKey, type TableEditorValues, type TableEditorValuesColumn } from '../tableEditor';
+export type AlterTableValues = {
+    name: string;
+    operations: AlterTableOperation[];
+};
+type AddColumnOperation = {
+    kind: 'addColumn';
+    column: TableEditorValuesColumn;
+};
+type DropColumnOperation = {
+    kind: 'dropColumn';
+    name: string;
+};
+export type AlterColumnOperation = {
+    kind: 'alterColumn';
+    original: TableEditorValuesColumn;
+    defaultConstraintName?: string;
+    changes: Partial<Omit<TableEditorValuesColumn, 'initialName'>>;
+};
+type AddForeignKeyOperation = {
+    kind: 'addForeignKey';
+    foreignKey: TableEditorForeignKey;
+};
+type DropForeignKeyOperation = {
+    kind: 'dropForeignKey';
+    fk_constraint_name: string;
+};
+type RenameTableOperation = {
+    kind: 'renameTable';
+    to: string;
+};
+type AddPrimaryKeyOperation = {
+    kind: 'addPrimaryKey';
+    columns: string[];
+};
+type DropPrimaryKeyOperation = {
+    kind: 'dropPrimaryKey';
+    pk_constraint_name?: string;
+};
+export type AlterTableOperation = AddColumnOperation | DropColumnOperation | AlterColumnOperation | AddForeignKeyOperation | DropForeignKeyOperation | RenameTableOperation | AddPrimaryKeyOperation | DropPrimaryKeyOperation;
+export declare function makeAlterTableQuery(values: AlterTableValues, dbType: DbType, schema?: string): string;
+export declare function makeAlterTableQueries(values: AlterTableValues, dbType: DbType, schema?: string): string[];
+export declare function diffTableEditorValues(original: TableEditorValues, updated: TableEditorValues): AlterTableValues;
+export declare function dbSupportsTransactionalDdl(dbType: DbType): boolean;
+export {};