yamchart 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +51 -0
- package/dist/{advisor-5BTRAICH.js → advisor-57BOMUUF.js} +11 -10
- package/dist/{advisor-5BTRAICH.js.map → advisor-57BOMUUF.js.map} +1 -1
- package/dist/{chunk-5VA43CTW.js → chunk-E2UZXDF6.js} +37 -42
- package/dist/chunk-E2UZXDF6.js.map +1 -0
- package/dist/{chunk-SSUVEADJ.js → chunk-JYQKDWLG.js} +76 -14
- package/dist/chunk-JYQKDWLG.js.map +1 -0
- package/dist/{chunk-565OEBW7.js → chunk-OTAUP5RC.js} +3 -3
- package/dist/chunk-UND73EOB.js +449 -0
- package/dist/chunk-UND73EOB.js.map +1 -0
- package/dist/{chunk-UDRJFEJU.js → chunk-X6LQGWUX.js} +349 -176
- package/dist/chunk-X6LQGWUX.js.map +1 -0
- package/dist/{connection-utils-AGSQ5HNN.js → connection-utils-FTSZU2VF.js} +5 -4
- package/dist/{describe-AA4UT4U6.js → describe-TGIOBNJB.js} +5 -4
- package/dist/{describe-AA4UT4U6.js.map → describe-TGIOBNJB.js.map} +1 -1
- package/dist/{dev-EUWIRL4N.js → dev-MRZ76V74.js} +485 -44
- package/dist/dev-MRZ76V74.js.map +1 -0
- package/dist/{dist-LZNDQDH3.js → dist-PVHFUYP2.js} +23 -3
- package/dist/{dist-YTGUIBKG.js → dist-WDTDQDTX.js} +72 -1
- package/dist/dist-WDTDQDTX.js.map +1 -0
- package/dist/index.js +135 -17
- package/dist/index.js.map +1 -1
- package/dist/{init-CI4VARQG.js → init-UYQE5DPU.js} +2 -2
- package/dist/{init-CI4VARQG.js.map → init-UYQE5DPU.js.map} +1 -1
- package/dist/public/assets/EventManagement-BlxJ2TFw.js +18 -0
- package/dist/public/assets/{LoginPage-BwMmpq8e.js → LoginPage-BT8ikmPn.js} +1 -1
- package/dist/public/assets/PublicViewer-B4uFxgbt.js +1 -0
- package/dist/public/assets/{SetupWizard-DU8R2dPp.js → SetupWizard-njrOhCzw.js} +1 -1
- package/dist/public/assets/ShareManagement-Bg16oJhW.js +1 -0
- package/dist/public/assets/UserManagement-tiCIT4UY.js +1 -0
- package/dist/public/assets/index-B41yj3io.js +187 -0
- package/dist/public/assets/{index-C0hWblBI.css → index-BZ25r23j.css} +1 -1
- package/dist/public/assets/{index.es-YeJujQ5o.js → index.es-BuktD_R2.js} +1 -1
- package/dist/public/assets/{jspdf.es.min-BzZgn3ka.js → jspdf.es.min-DFRl2hZQ.js} +3 -3
- package/dist/public/index.html +2 -2
- package/dist/{query-THDVBBVZ.js → query-JRMMNXX6.js} +5 -4
- package/dist/{query-THDVBBVZ.js.map → query-JRMMNXX6.js.map} +1 -1
- package/dist/{sample-6WPQS7PA.js → sample-VGIY4U4J.js} +5 -4
- package/dist/{sample-6WPQS7PA.js.map → sample-VGIY4U4J.js.map} +1 -1
- package/dist/{search-657DXBRS.js → search-QSYNG4SR.js} +5 -4
- package/dist/{search-657DXBRS.js.map → search-QSYNG4SR.js.map} +1 -1
- package/dist/semantic-RAP3S5PQ.js +39 -0
- package/dist/semantic-RAP3S5PQ.js.map +1 -0
- package/dist/{sync-warehouse-4KBV5S3L.js → sync-warehouse-XHTBZH25.js} +5 -4
- package/dist/{sync-warehouse-4KBV5S3L.js.map → sync-warehouse-XHTBZH25.js.map} +1 -1
- package/dist/{tables-KLDBUUSE.js → tables-UOO342TA.js} +5 -4
- package/dist/{tables-KLDBUUSE.js.map → tables-UOO342TA.js.map} +1 -1
- package/dist/templates/default/.claude/skills/databricks/SKILL.md +76 -0
- package/dist/templates/default/.claude/skills/dbt-advisor/SKILL.md +107 -0
- package/dist/templates/default/.claude/skills/duckdb/SKILL.md +67 -0
- package/dist/templates/default/.claude/skills/mysql/SKILL.md +69 -0
- package/dist/templates/default/.claude/skills/postgres/SKILL.md +69 -0
- package/dist/templates/default/.claude/skills/snowflake/SKILL.md +64 -0
- package/dist/templates/default/CLAUDE.md +7 -0
- package/dist/templates/default/docs/yamchart-reference.md +222 -1
- package/dist/{test-JSAWS5ZP.js → test-TXRZWNXK.js} +5 -4
- package/dist/{test-JSAWS5ZP.js.map → test-TXRZWNXK.js.map} +1 -1
- package/dist/update-UKMEWCSO.js +220 -0
- package/dist/update-UKMEWCSO.js.map +1 -0
- package/package.json +5 -3
- package/dist/chunk-5VA43CTW.js.map +0 -1
- package/dist/chunk-SSUVEADJ.js.map +0 -1
- package/dist/chunk-UDRJFEJU.js.map +0 -1
- package/dist/dev-EUWIRL4N.js.map +0 -1
- package/dist/dist-YTGUIBKG.js.map +0 -1
- package/dist/public/assets/PublicViewer-vIDojjIR.js +0 -1
- package/dist/public/assets/ShareManagement-7iS6lM2T.js +0 -1
- package/dist/public/assets/UserManagement-By7YRZrF.js +0 -1
- package/dist/public/assets/index-CXx1PiRF.js +0 -174
- package/dist/update-HCR6MYJX.js +0 -88
- package/dist/update-HCR6MYJX.js.map +0 -1
- /package/dist/{chunk-565OEBW7.js.map → chunk-OTAUP5RC.js.map} +0 -0
- /package/dist/{connection-utils-AGSQ5HNN.js.map → connection-utils-FTSZU2VF.js.map} +0 -0
- /package/dist/{dist-LZNDQDH3.js.map → dist-PVHFUYP2.js.map} +0 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/commands/warehouse-sync.ts","../src/commands/sync-warehouse.ts"],"sourcesContent":["/**\n * Core warehouse sync logic.\n *\n * Pure functions for diffing, grouping, building models, and merging catalogs,\n * plus an async orchestrator (syncWarehouse) that does I/O.\n */\n\nimport { readFile, writeFile, mkdir } from 'fs/promises';\nimport { join } from 'path';\nimport type { Connector } from '@yamchart/query';\nimport type { NormalizedColumnWithTable } from './introspection.js';\nimport { getTablesQuery, getColumnsQuery, getDescribeQuery } from './introspection.js';\nimport type { CatalogModel, CatalogData } from '../dbt/catalog.js';\nimport { generateCatalogMd, generateCatalogJson, generateWarehouseSection } from '../dbt/catalog.js';\n\n// ---------------------------------------------------------------------------\n// Interfaces\n// ---------------------------------------------------------------------------\n\nexport interface SyncState {\n syncedAt: string;\n connection: string;\n schemas: string[];\n skipSamples: boolean;\n tables: Record<string, { columnCount: number; lastAltered?: string }>;\n}\n\nexport interface WarehouseTable {\n qualifiedName: string; // \"SCHEMA.TABLE\"\n schema: string;\n name: string;\n type: string; // \"TABLE\" | \"VIEW\"\n columnCount: number;\n}\n\nexport interface SyncDiff {\n newTables: string[];\n changedTables: string[];\n unchangedTables: string[];\n removedTables: string[];\n}\n\nexport interface SyncWarehouseResult {\n tablesTotal: number;\n tablesNew: number;\n tablesChanged: number;\n tablesUnchanged: number;\n tablesRemoved: number;\n errors: string[];\n durationMs: number;\n}\n\nexport interface SyncWarehouseOptions {\n schemas?: string[];\n database?: string;\n skipSamples?: boolean;\n full?: boolean;\n onProgress?: (table: string, index: number, total: number) => void;\n}\n\n// ---------------------------------------------------------------------------\n// Pure functions\n// ---------------------------------------------------------------------------\n\n/**\n * Group a flat column list into a Map keyed by \"SCHEMA.TABLE\".\n */\nexport function groupColumnsByTable(\n columns: NormalizedColumnWithTable[],\n): Map<string, NormalizedColumnWithTable[]> {\n const map = new Map<string, NormalizedColumnWithTable[]>();\n for (const col of columns) {\n const key = `${col.schema}.${col.table}`;\n const existing = map.get(key);\n if (existing) {\n existing.push(col);\n } else {\n map.set(key, [col]);\n }\n }\n return map;\n}\n\n/**\n * Compare current warehouse tables against previous sync state.\n * All tables are \"new\" when prev is null.\n * Detects changed by column count mismatch.\n * Detects removed tables (in prev but not in current).\n */\nexport function diffSyncState(\n current: WarehouseTable[],\n prev: SyncState | null,\n): SyncDiff {\n const newTables: string[] = [];\n const changedTables: string[] = [];\n const unchangedTables: string[] = [];\n const removedTables: string[] = [];\n\n if (!prev) {\n return {\n newTables: current.map((t) => t.qualifiedName),\n changedTables: [],\n unchangedTables: [],\n removedTables: [],\n };\n }\n\n const currentNames = new Set(current.map((t) => t.qualifiedName));\n\n for (const table of current) {\n const prevEntry = prev.tables[table.qualifiedName];\n if (!prevEntry) {\n newTables.push(table.qualifiedName);\n } else if (prevEntry.columnCount !== table.columnCount) {\n changedTables.push(table.qualifiedName);\n } else {\n unchangedTables.push(table.qualifiedName);\n }\n }\n\n // Detect removed tables\n for (const name of Object.keys(prev.tables)) {\n if (!currentNames.has(name)) {\n removedTables.push(name);\n }\n }\n\n return { newTables, changedTables, unchangedTables, removedTables };\n}\n\n/**\n * Build CatalogModel entries from warehouse metadata.\n */\nexport function buildWarehouseModels(\n tables: WarehouseTable[],\n columnMap: Map<string, NormalizedColumnWithTable[]>,\n samples: Record<string, Record<string, unknown>[]>,\n): CatalogModel[] {\n const now = new Date().toISOString();\n\n return tables.map((table) => {\n const cols = columnMap.get(table.qualifiedName) || [];\n const sampleRows = samples[table.qualifiedName];\n\n return {\n name: table.name,\n path: '',\n description: '',\n table: table.qualifiedName,\n tags: ['warehouse'],\n meta: {},\n columns: cols.map((c) => ({\n name: c.name,\n data_type: c.type,\n description: '',\n hints: [],\n })),\n yamchartModels: [],\n source: 'warehouse' as const,\n tableType: table.type as 'TABLE' | 'VIEW',\n ...(sampleRows ? { sampleRows } : {}),\n lastSynced: now,\n };\n });\n}\n\n/**\n * Merge warehouse models into an existing dbt catalog.\n *\n * 1. If no existing catalog, create new with warehouseModels.\n * 2. Remove stale warehouse entries from existing.\n * 3. For tables in both dbt and warehouse (matched by `table` field, case-insensitive):\n * - dbt entry wins (keep it)\n * - Backfill data_type from warehouse columns where missing\n * - Add any warehouse columns that don't exist in the dbt entry\n * 4. Append remaining warehouse-only models.\n * 5. Return updated CatalogData.\n */\nexport function mergeWithDbtCatalog(\n existingCatalog: CatalogData | null,\n warehouseModels: CatalogModel[],\n): CatalogData {\n if (!existingCatalog) {\n return {\n syncedAt: new Date().toISOString(),\n source: { type: 'warehouse' },\n stats: { modelsIncluded: warehouseModels.length, modelsExcluded: 0 },\n models: warehouseModels,\n };\n }\n\n // Build a lookup of warehouse models by normalized table name\n const warehouseByTable = new Map<string, CatalogModel>();\n for (const wm of warehouseModels) {\n if (wm.table) {\n warehouseByTable.set(wm.table.toUpperCase(), wm);\n }\n }\n\n // Track which warehouse models were consumed by dbt merge\n const consumedWarehouseTables = new Set<string>();\n\n // Process existing catalog: keep non-warehouse entries, merge dbt+warehouse matches\n const mergedModels: CatalogModel[] = [];\n\n for (const model of existingCatalog.models) {\n // Remove stale warehouse entries — they'll be replaced by current warehouseModels\n if (model.source === 'warehouse') {\n continue;\n }\n\n // Check if this dbt model matches a warehouse table\n const tableKey = (model.table || '').toUpperCase();\n const warehouseMatch = tableKey ? warehouseByTable.get(tableKey) : undefined;\n\n if (warehouseMatch) {\n // dbt entry wins — backfill data_type and add missing columns\n const warehouseColMap = new Map<string, { data_type?: string; description: string; hints: string[] }>();\n for (const wc of warehouseMatch.columns) {\n warehouseColMap.set(wc.name.toLowerCase(), wc);\n }\n\n // Backfill data_type on existing dbt columns\n const updatedColumns = model.columns.map((col) => {\n const wc = warehouseColMap.get(col.name.toLowerCase());\n if (wc && !col.data_type && wc.data_type) {\n return { ...col, data_type: wc.data_type };\n }\n return col;\n });\n\n // Add warehouse columns not present in dbt entry\n const existingColNames = new Set(model.columns.map((c) => c.name.toLowerCase()));\n for (const wc of warehouseMatch.columns) {\n if (!existingColNames.has(wc.name.toLowerCase())) {\n updatedColumns.push(wc);\n }\n }\n\n mergedModels.push({ ...model, columns: updatedColumns });\n consumedWarehouseTables.add(tableKey);\n } else {\n mergedModels.push(model);\n }\n }\n\n // Append warehouse-only models (not consumed by dbt merge)\n for (const wm of warehouseModels) {\n const tableKey = (wm.table || '').toUpperCase();\n if (!consumedWarehouseTables.has(tableKey)) {\n mergedModels.push(wm);\n }\n }\n\n return {\n ...existingCatalog,\n syncedAt: new Date().toISOString(),\n models: mergedModels,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Async orchestrator\n// ---------------------------------------------------------------------------\n\n/**\n * Main warehouse sync orchestrator.\n *\n * 1. Fetch tables and columns from the warehouse.\n * 2. Diff against previous sync state.\n * 3. Sample new/changed tables (unless skipSamples).\n * 4. Build warehouse CatalogModel entries.\n * 5. Merge with existing catalog, write files.\n */\nexport async function syncWarehouse(\n projectDir: string,\n connector: Connector,\n connectionType: string,\n connectionName: string,\n options?: SyncWarehouseOptions,\n): Promise<SyncWarehouseResult> {\n const start = performance.now();\n const errors: string[] = [];\n\n // 1. Fetch tables\n const tablesQuery = getTablesQuery(connectionType, {\n database: options?.database,\n });\n const tablesResult = await connector.execute(tablesQuery.sql);\n let normalizedTables = tablesQuery.normalize(tablesResult.rows);\n\n // Filter by schemas if specified\n if (options?.schemas && options.schemas.length > 0) {\n const schemaSet = new Set(options.schemas.map((s) => s.toUpperCase()));\n normalizedTables = normalizedTables.filter((t) => schemaSet.has(t.schema.toUpperCase()));\n }\n\n // 2. Fetch columns\n let allColumns: NormalizedColumnWithTable[] = [];\n\n if (connectionType === 'sqlite') {\n // SQLite: describe each table individually\n for (const table of normalizedTables) {\n try {\n const describeQuery = getDescribeQuery('sqlite', table.name);\n const descResult = await connector.execute(describeQuery.sql);\n const cols = describeQuery.normalize(descResult.rows);\n for (const col of cols) {\n allColumns.push({\n schema: table.schema,\n table: table.name,\n name: col.name,\n type: col.type,\n nullable: col.nullable,\n });\n }\n } catch (err) {\n errors.push(`Failed to describe ${table.name}: ${err instanceof Error ? err.message : String(err)}`);\n }\n }\n } else {\n const columnsQuery = getColumnsQuery(connectionType, {\n database: options?.database,\n });\n const columnsResult = await connector.execute(columnsQuery.sql);\n allColumns = columnsQuery.normalize(columnsResult.rows);\n\n // Filter columns by schemas if specified\n if (options?.schemas && options.schemas.length > 0) {\n const schemaSet = new Set(options.schemas.map((s) => s.toUpperCase()));\n allColumns = allColumns.filter((c) => schemaSet.has(c.schema.toUpperCase()));\n }\n }\n\n // Group columns by table\n const columnMap = groupColumnsByTable(allColumns);\n\n // Build WarehouseTable entries with column counts\n const warehouseTables: WarehouseTable[] = normalizedTables.map((t) => {\n const qualifiedName = t.schema ? `${t.schema}.${t.name}` : t.name;\n return {\n qualifiedName,\n schema: t.schema,\n name: t.name,\n type: t.type,\n columnCount: columnMap.get(qualifiedName)?.length || 0,\n };\n });\n\n // 3. Load previous sync state, compute diff\n let prevState: SyncState | null = null;\n if (!options?.full) {\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), 'utf-8');\n prevState = JSON.parse(raw);\n } catch {\n /* no previous state */\n }\n }\n\n const diff = diffSyncState(warehouseTables, prevState);\n\n // 4. Sample new/changed tables (unless skipSamples)\n const samples: Record<string, Record<string, unknown>[]> = {};\n const tablesToSample = options?.skipSamples\n ? []\n : [...diff.newTables, ...diff.changedTables];\n\n for (let i = 0; i < tablesToSample.length; i++) {\n const tableName = tablesToSample[i]!;\n options?.onProgress?.(tableName, i + 1, tablesToSample.length);\n\n try {\n const sampleSql = `SELECT * FROM ${tableName} LIMIT 3`;\n const samplePromise = connector.execute(sampleSql);\n const timeoutPromise = new Promise<never>((_, reject) =>\n setTimeout(() => reject(new Error(`Timeout sampling ${tableName} (5s)`)), 5000),\n );\n const result = await Promise.race([samplePromise, timeoutPromise]);\n samples[tableName] = result.rows;\n } catch (err) {\n errors.push(`Failed to sample ${tableName}: ${err instanceof Error ? err.message : String(err)}`);\n }\n }\n\n // 5. For unchanged tables: carry forward sampleRows from existing catalog\n let existingCatalog: CatalogData | null = null;\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'catalog.json'), 'utf-8');\n existingCatalog = JSON.parse(raw);\n } catch {\n /* no existing catalog */\n }\n\n if (existingCatalog && !options?.skipSamples) {\n for (const tableName of diff.unchangedTables) {\n const existingModel = existingCatalog.models.find(\n (m) => m.table?.toUpperCase() === tableName.toUpperCase() && m.source === 'warehouse',\n );\n if (existingModel?.sampleRows) {\n samples[tableName] = existingModel.sampleRows;\n }\n }\n }\n\n // 6. Build warehouse CatalogModel entries\n const warehouseModels = buildWarehouseModels(warehouseTables, columnMap, samples);\n\n // 7. Merge with existing catalog, write files\n const merged = mergeWithDbtCatalog(existingCatalog, warehouseModels);\n\n await mkdir(join(projectDir, '.yamchart'), { recursive: true });\n await writeFile(join(projectDir, '.yamchart', 'catalog.json'), generateCatalogJson(merged));\n\n const catalogMd = generateCatalogMd(merged);\n const syncDate = new Date().toISOString().split('T')[0]!;\n const warehouseSection = generateWarehouseSection(merged.models, connectionName, syncDate);\n const fullMd = warehouseSection ? catalogMd + '\\n' + warehouseSection : catalogMd;\n await writeFile(join(projectDir, '.yamchart', 'catalog.md'), fullMd);\n\n // Write sync state\n const syncState: SyncState = {\n syncedAt: new Date().toISOString(),\n connection: connectionName,\n schemas: options?.schemas || [],\n skipSamples: options?.skipSamples || false,\n tables: Object.fromEntries(\n warehouseTables.map((t) => [t.qualifiedName, { columnCount: t.columnCount }]),\n ),\n };\n await writeFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), JSON.stringify(syncState, null, 2));\n\n const durationMs = Math.round(performance.now() - start);\n\n return {\n tablesTotal: warehouseTables.length,\n tablesNew: diff.newTables.length,\n tablesChanged: diff.changedTables.length,\n tablesUnchanged: diff.unchangedTables.length,\n tablesRemoved: diff.removedTables.length,\n errors,\n durationMs,\n };\n}\n","/**\n * CLI handler for `yamchart sync-warehouse`.\n *\n * Thin wrapper: resolves connection, creates connector, delegates to syncWarehouse().\n */\n\nimport * as output from '../utils/output.js';\nimport { resolveConnection, createConnector } from './connection-utils.js';\nimport { syncWarehouse } from './warehouse-sync.js';\nimport { readFile } from 'fs/promises';\nimport { join } from 'path';\nimport type { SyncState } from './warehouse-sync.js';\n\nexport interface SyncWarehouseCliOptions {\n connection?: string;\n schema?: string; // comma-separated → split into array\n database?: string;\n skipSamples?: boolean;\n refresh?: boolean;\n full?: boolean;\n json?: boolean;\n}\n\nexport async function runSyncWarehouse(\n projectDir: string,\n options: SyncWarehouseCliOptions,\n): Promise<void> {\n // Handle --refresh: load saved connection/schemas from previous sync state\n let connectionOverride = options.connection;\n let schemasOverride = options.schema;\n\n if (options.refresh) {\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), 'utf-8');\n const prevState: SyncState = JSON.parse(raw);\n connectionOverride = connectionOverride || prevState.connection;\n if (!schemasOverride && prevState.schemas.length > 0) {\n schemasOverride = prevState.schemas.join(',');\n }\n if (!options.json) {\n output.info(`Re-syncing from ${prevState.connection} (${prevState.schemas.join(', ') || 'all schemas'})`);\n }\n } catch {\n output.error('No previous sync state found. Run sync-warehouse without --refresh first.');\n process.exit(1);\n }\n }\n\n // Resolve connection\n const connection = await resolveConnection(projectDir, connectionOverride);\n const connector = createConnector(connection, projectDir);\n\n const schemas = schemasOverride\n ? schemasOverride.split(',').map((s) => s.trim())\n : undefined;\n\n const spin = options.json ? null : output.spinner(`Connecting to ${connection.name} ...`);\n\n try {\n await connector.connect();\n\n if (spin) {\n spin.text = 'Fetching table metadata...';\n }\n\n const result = await syncWarehouse(\n projectDir,\n connector,\n connection.type,\n connection.name,\n {\n schemas,\n database: options.database,\n skipSamples: options.skipSamples,\n full: options.full,\n onProgress: spin\n ? (table, index, total) => {\n spin.text = `Sampling tables... ${index}/${total} (${table})`;\n }\n : undefined,\n },\n );\n\n if (spin) {\n spin.stop();\n }\n\n if (options.json) {\n console.log(JSON.stringify(result, null, 2));\n } else {\n const parts = [\n `${result.tablesNew} new`,\n `${result.tablesChanged} changed`,\n `${result.tablesUnchanged} unchanged`,\n ];\n if (result.tablesRemoved > 0) {\n parts.push(`${result.tablesRemoved} removed`);\n }\n output.success(\n `Synced ${result.tablesTotal} tables (${parts.join(', ')}) in ${(result.durationMs / 1000).toFixed(1)}s`,\n );\n output.detail('Written to .yamchart/catalog.json');\n\n if (result.errors.length > 0) {\n output.warning(`${result.errors.length} table(s) had sampling errors:`);\n for (const err of result.errors.slice(0, 5)) {\n output.detail(err);\n }\n if (result.errors.length > 5) {\n output.detail(`... and ${result.errors.length - 5} more`);\n }\n }\n }\n } finally {\n await connector.disconnect();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA,SAAS,UAAU,WAAW,aAAa;AAC3C,SAAS,YAAY;AA2Dd,SAAS,oBACd,SAC0C;AAC1C,QAAM,MAAM,oBAAI,IAAyC;AACzD,aAAW,OAAO,SAAS;AACzB,UAAM,MAAM,GAAG,IAAI,MAAM,IAAI,IAAI,KAAK;AACtC,UAAM,WAAW,IAAI,IAAI,GAAG;AAC5B,QAAI,UAAU;AACZ,eAAS,KAAK,GAAG;AAAA,IACnB,OAAO;AACL,UAAI,IAAI,KAAK,CAAC,GAAG,CAAC;AAAA,IACpB;AAAA,EACF;AACA,SAAO;AACT;AAQO,SAAS,cACd,SACA,MACU;AACV,QAAM,YAAsB,CAAC;AAC7B,QAAM,gBAA0B,CAAC;AACjC,QAAM,kBAA4B,CAAC;AACnC,QAAM,gBAA0B,CAAC;AAEjC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,MACL,WAAW,QAAQ,IAAI,CAAC,MAAM,EAAE,aAAa;AAAA,MAC7C,eAAe,CAAC;AAAA,MAChB,iBAAiB,CAAC;AAAA,MAClB,eAAe,CAAC;AAAA,IAClB;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC;AAEhE,aAAW,SAAS,SAAS;AAC3B,UAAM,YAAY,KAAK,OAAO,MAAM,aAAa;AACjD,QAAI,CAAC,WAAW;AACd,gBAAU,KAAK,MAAM,aAAa;AAAA,IACpC,WAAW,UAAU,gBAAgB,MAAM,aAAa;AACtD,oBAAc,KAAK,MAAM,aAAa;AAAA,IACxC,OAAO;AACL,sBAAgB,KAAK,MAAM,aAAa;AAAA,IAC1C;AAAA,EACF;AAGA,aAAW,QAAQ,OAAO,KAAK,KAAK,MAAM,GAAG;AAC3C,QAAI,CAAC,aAAa,IAAI,IAAI,GAAG;AAC3B,oBAAc,KAAK,IAAI;AAAA,IACzB;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,eAAe,iBAAiB,cAAc;AACpE;AAKO,SAAS,qBACd,QACA,WACA,SACgB;AAChB,QAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AAEnC,SAAO,OAAO,IAAI,CAAC,UAAU;AAC3B,UAAM,OAAO,UAAU,IAAI,MAAM,aAAa,KAAK,CAAC;AACpD,UAAM,aAAa,QAAQ,MAAM,aAAa;AAE9C,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,MAAM;AAAA,MACN,aAAa;AAAA,MACb,OAAO,MAAM;AAAA,MACb,MAAM,CAAC,WAAW;AAAA,MAClB,MAAM,CAAC;AAAA,MACP,SAAS,KAAK,IAAI,CAAC,OAAO;AAAA,QACxB,MAAM,EAAE;AAAA,QACR,WAAW,EAAE;AAAA,QACb,aAAa;AAAA,QACb,OAAO,CAAC;AAAA,MACV,EAAE;AAAA,MACF,gBAAgB,CAAC;AAAA,MACjB,QAAQ;AAAA,MACR,WAAW,MAAM;AAAA,MACjB,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,MACnC,YAAY;AAAA,IACd;AAAA,EACF,CAAC;AACH;AAcO,SAAS,oBACd,iBACA,iBACa;AACb,MAAI,CAAC,iBAAiB;AACpB,WAAO;AAAA,MACL,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,MACjC,QAAQ,EAAE,MAAM,YAAY;AAAA,MAC5B,OAAO,EAAE,gBAAgB,gBAAgB,QAAQ,gBAAgB,EAAE;AAAA,MACnE,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,QAAM,mBAAmB,oBAAI,IAA0B;AACvD,aAAW,MAAM,iBAAiB;AAChC,QAAI,GAAG,OAAO;AACZ,uBAAiB,IAAI,GAAG,MAAM,YAAY,GAAG,EAAE;AAAA,IACjD;AAAA,EACF;AAGA,QAAM,0BAA0B,oBAAI,IAAY;AAGhD,QAAM,eAA+B,CAAC;AAEtC,aAAW,SAAS,gBAAgB,QAAQ;AAE1C,QAAI,MAAM,WAAW,aAAa;AAChC;AAAA,IACF;AAGA,UAAM,YAAY,MAAM,SAAS,IAAI,YAAY;AACjD,UAAM,iBAAiB,WAAW,iBAAiB,IAAI,QAAQ,IAAI;AAEnE,QAAI,gBAAgB;AAElB,YAAM,kBAAkB,oBAAI,IAA0E;AACtG,iBAAW,MAAM,eAAe,SAAS;AACvC,wBAAgB,IAAI,GAAG,KAAK,YAAY,GAAG,EAAE;AAAA,MAC/C;AAGA,YAAM,iBAAiB,MAAM,QAAQ,IAAI,CAAC,QAAQ;AAChD,cAAM,KAAK,gBAAgB,IAAI,IAAI,KAAK,YAAY,CAAC;AACrD,YAAI,MAAM,CAAC,IAAI,aAAa,GAAG,WAAW;AACxC,iBAAO,EAAE,GAAG,KAAK,WAAW,GAAG,UAAU;AAAA,QAC3C;AACA,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,mBAAmB,IAAI,IAAI,MAAM,QAAQ,IAAI,CAAC,MAAM,EAAE,KAAK,YAAY,CAAC,CAAC;AAC/E,iBAAW,MAAM,eAAe,SAAS;AACvC,YAAI,CAAC,iBAAiB,IAAI,GAAG,KAAK,YAAY,CAAC,GAAG;AAChD,yBAAe,KAAK,EAAE;AAAA,QACxB;AAAA,MACF;AAEA,mBAAa,KAAK,EAAE,GAAG,OAAO,SAAS,eAAe,CAAC;AACvD,8BAAwB,IAAI,QAAQ;AAAA,IACtC,OAAO;AACL,mBAAa,KAAK,KAAK;AAAA,IACzB;AAAA,EACF;AAGA,aAAW,MAAM,iBAAiB;AAChC,UAAM,YAAY,GAAG,SAAS,IAAI,YAAY;AAC9C,QAAI,CAAC,wBAAwB,IAAI,QAAQ,GAAG;AAC1C,mBAAa,KAAK,EAAE;AAAA,IACtB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG;AAAA,IACH,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,IACjC,QAAQ;AAAA,EACV;AACF;AAeA,eAAsB,cACpB,YACA,WACA,gBACA,gBACA,SAC8B;AAC9B,QAAM,QAAQ,YAAY,IAAI;AAC9B,QAAM,SAAmB,CAAC;AAG1B,QAAM,cAAc,eAAe,gBAAgB;AAAA,IACjD,UAAU,SAAS;AAAA,EACrB,CAAC;AACD,QAAM,eAAe,MAAM,UAAU,QAAQ,YAAY,GAAG;AAC5D,MAAI,mBAAmB,YAAY,UAAU,aAAa,IAAI;AAG9D,MAAI,SAAS,WAAW,QAAQ,QAAQ,SAAS,GAAG;AAClD,UAAM,YAAY,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACrE,uBAAmB,iBAAiB,OAAO,CAAC,MAAM,UAAU,IAAI,EAAE,OAAO,YAAY,CAAC,CAAC;AAAA,EACzF;AAGA,MAAI,aAA0C,CAAC;AAE/C,MAAI,mBAAmB,UAAU;AAE/B,eAAW,SAAS,kBAAkB;AACpC,UAAI;AACF,cAAM,gBAAgB,iBAAiB,UAAU,MAAM,IAAI;AAC3D,cAAM,aAAa,MAAM,UAAU,QAAQ,cAAc,GAAG;AAC5D,cAAM,OAAO,cAAc,UAAU,WAAW,IAAI;AACpD,mBAAW,OAAO,MAAM;AACtB,qBAAW,KAAK;AAAA,YACd,QAAQ,MAAM;AAAA,YACd,OAAO,MAAM;AAAA,YACb,MAAM,IAAI;AAAA,YACV,MAAM,IAAI;AAAA,YACV,UAAU,IAAI;AAAA,UAChB,CAAC;AAAA,QACH;AAAA,MACF,SAAS,KAAK;AACZ,eAAO,KAAK,sBAAsB,MAAM,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,MACrG;AAAA,IACF;AAAA,EACF,OAAO;AACL,UAAM,eAAe,gBAAgB,gBAAgB;AAAA,MACnD,UAAU,SAAS;AAAA,IACrB,CAAC;AACD,UAAM,gBAAgB,MAAM,UAAU,QAAQ,aAAa,GAAG;AAC9D,iBAAa,aAAa,UAAU,cAAc,IAAI;AAGtD,QAAI,SAAS,WAAW,QAAQ,QAAQ,SAAS,GAAG;AAClD,YAAM,YAAY,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACrE,mBAAa,WAAW,OAAO,CAAC,MAAM,UAAU,IAAI,EAAE,OAAO,YAAY,CAAC,CAAC;AAAA,IAC7E;AAAA,EACF;AAGA,QAAM,YAAY,oBAAoB,UAAU;AAGhD,QAAM,kBAAoC,iBAAiB,IAAI,CAAC,MAAM;AACpE,UAAM,gBAAgB,EAAE,SAAS,GAAG,EAAE,MAAM,IAAI,EAAE,IAAI,KAAK,EAAE;AAC7D,WAAO;AAAA,MACL;AAAA,MACA,QAAQ,EAAE;AAAA,MACV,MAAM,EAAE;AAAA,MACR,MAAM,EAAE;AAAA,MACR,aAAa,UAAU,IAAI,aAAa,GAAG,UAAU;AAAA,IACvD;AAAA,EACF,CAAC;AAGD,MAAI,YAA8B;AAClC,MAAI,CAAC,SAAS,MAAM;AAClB,QAAI;AACF,YAAM,MAAM,MAAM,SAAS,KAAK,YAAY,aAAa,qBAAqB,GAAG,OAAO;AACxF,kBAAY,KAAK,MAAM,GAAG;AAAA,IAC5B,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,QAAM,OAAO,cAAc,iBAAiB,SAAS;AAGrD,QAAM,UAAqD,CAAC;AAC5D,QAAM,iBAAiB,SAAS,cAC5B,CAAC,IACD,CAAC,GAAG,KAAK,WAAW,GAAG,KAAK,aAAa;AAE7C,WAAS,IAAI,GAAG,IAAI,eAAe,QAAQ,KAAK;AAC9C,UAAM,YAAY,eAAe,CAAC;AAClC,aAAS,aAAa,WAAW,IAAI,GAAG,eAAe,MAAM;AAE7D,QAAI;AACF,YAAM,YAAY,iBAAiB,SAAS;AAC5C,YAAM,gBAAgB,UAAU,QAAQ,SAAS;AACjD,YAAM,iBAAiB,IAAI;AAAA,QAAe,CAAC,GAAG,WAC5C,WAAW,MAAM,OAAO,IAAI,MAAM,oBAAoB,SAAS,OAAO,CAAC,GAAG,GAAI;AAAA,MAChF;AACA,YAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,eAAe,cAAc,CAAC;AACjE,cAAQ,SAAS,IAAI,OAAO;AAAA,IAC9B,SAAS,KAAK;AACZ,aAAO,KAAK,oBAAoB,SAAS,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,IAClG;AAAA,EACF;AAGA,MAAI,kBAAsC;AAC1C,MAAI;AACF,UAAM,MAAM,MAAM,SAAS,KAAK,YAAY,aAAa,cAAc,GAAG,OAAO;AACjF,sBAAkB,KAAK,MAAM,GAAG;AAAA,EAClC,QAAQ;AAAA,EAER;AAEA,MAAI,mBAAmB,CAAC,SAAS,aAAa;AAC5C,eAAW,aAAa,KAAK,iBAAiB;AAC5C,YAAM,gBAAgB,gBAAgB,OAAO;AAAA,QAC3C,CAAC,MAAM,EAAE,OAAO,YAAY,MAAM,UAAU,YAAY,KAAK,EAAE,WAAW;AAAA,MAC5E;AACA,UAAI,eAAe,YAAY;AAC7B,gBAAQ,SAAS,IAAI,cAAc;AAAA,MACrC;AAAA,IACF;AAAA,EACF;AAGA,QAAM,kBAAkB,qBAAqB,iBAAiB,WAAW,OAAO;AAGhF,QAAM,SAAS,oBAAoB,iBAAiB,eAAe;AAEnE,QAAM,MAAM,KAAK,YAAY,WAAW,GAAG,EAAE,WAAW,KAAK,CAAC;AAC9D,QAAM,UAAU,KAAK,YAAY,aAAa,cAAc,GAAG,oBAAoB,MAAM,CAAC;AAE1F,QAAM,YAAY,kBAAkB,MAAM;AAC1C,QAAM,YAAW,oBAAI,KAAK,GAAE,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC;AACtD,QAAM,mBAAmB,yBAAyB,OAAO,QAAQ,gBAAgB,QAAQ;AACzF,QAAM,SAAS,mBAAmB,YAAY,OAAO,mBAAmB;AACxE,QAAM,UAAU,KAAK,YAAY,aAAa,YAAY,GAAG,MAAM;AAGnE,QAAM,YAAuB;AAAA,IAC3B,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,IACjC,YAAY;AAAA,IACZ,SAAS,SAAS,WAAW,CAAC;AAAA,IAC9B,aAAa,SAAS,eAAe;AAAA,IACrC,QAAQ,OAAO;AAAA,MACb,gBAAgB,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;AAAA,IAC9E;AAAA,EACF;AACA,QAAM,UAAU,KAAK,YAAY,aAAa,qBAAqB,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;AAExG,QAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AAEvD,SAAO;AAAA,IACL,aAAa,gBAAgB;AAAA,IAC7B,WAAW,KAAK,UAAU;AAAA,IAC1B,eAAe,KAAK,cAAc;AAAA,IAClC,iBAAiB,KAAK,gBAAgB;AAAA,IACtC,eAAe,KAAK,cAAc;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;;;AClbA,SAAS,YAAAA,iBAAgB;AACzB,SAAS,QAAAC,aAAY;AAarB,eAAsB,iBACpB,YACA,SACe;AAEf,MAAI,qBAAqB,QAAQ;AACjC,MAAI,kBAAkB,QAAQ;AAE9B,MAAI,QAAQ,SAAS;AACnB,QAAI;AACF,YAAM,MAAM,MAAMD,UAASC,MAAK,YAAY,aAAa,qBAAqB,GAAG,OAAO;AACxF,YAAM,YAAuB,KAAK,MAAM,GAAG;AAC3C,2BAAqB,sBAAsB,UAAU;AACrD,UAAI,CAAC,mBAAmB,UAAU,QAAQ,SAAS,GAAG;AACpD,0BAAkB,UAAU,QAAQ,KAAK,GAAG;AAAA,MAC9C;AACA,UAAI,CAAC,QAAQ,MAAM;AACjB,QAAO,KAAK,mBAAmB,UAAU,UAAU,KAAK,UAAU,QAAQ,KAAK,IAAI,KAAK,aAAa,GAAG;AAAA,MAC1G;AAAA,IACF,QAAQ;AACN,MAAO,MAAM,2EAA2E;AACxF,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AAGA,QAAM,aAAa,MAAM,kBAAkB,YAAY,kBAAkB;AACzE,QAAM,YAAY,gBAAgB,YAAY,UAAU;AAExD,QAAM,UAAU,kBACZ,gBAAgB,MAAM,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAC9C;AAEJ,QAAM,OAAO,QAAQ,OAAO,OAAc,QAAQ,iBAAiB,WAAW,IAAI,MAAM;AAExF,MAAI;AACF,UAAM,UAAU,QAAQ;AAExB,QAAI,MAAM;AACR,WAAK,OAAO;AAAA,IACd;AAEA,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,WAAW;AAAA,MACX;AAAA,QACE;AAAA,QACA,UAAU,QAAQ;AAAA,QAClB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,QACd,YAAY,OACR,CAAC,OAAO,OAAO,UAAU;AACvB,eAAK,OAAO,sBAAsB,KAAK,IAAI,KAAK,KAAK,KAAK;AAAA,QAC5D,IACA;AAAA,MACN;AAAA,IACF;AAEA,QAAI,MAAM;AACR,WAAK,KAAK;AAAA,IACZ;AAEA,QAAI,QAAQ,MAAM;AAChB,cAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,IAC7C,OAAO;AACL,YAAM,QAAQ;AAAA,QACZ,GAAG,OAAO,SAAS;AAAA,QACnB,GAAG,OAAO,aAAa;AAAA,QACvB,GAAG,OAAO,eAAe;AAAA,MAC3B;AACA,UAAI,OAAO,gBAAgB,GAAG;AAC5B,cAAM,KAAK,GAAG,OAAO,aAAa,UAAU;AAAA,MAC9C;AACA,MAAO;AAAA,QACL,UAAU,OAAO,WAAW,YAAY,MAAM,KAAK,IAAI,CAAC,SAAS,OAAO,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,MACvG;AACA,MAAO,OAAO,mCAAmC;AAEjD,UAAI,OAAO,OAAO,SAAS,GAAG;AAC5B,QAAO,QAAQ,GAAG,OAAO,OAAO,MAAM,gCAAgC;AACtE,mBAAW,OAAO,OAAO,OAAO,MAAM,GAAG,CAAC,GAAG;AAC3C,UAAO,OAAO,GAAG;AAAA,QACnB;AACA,YAAI,OAAO,OAAO,SAAS,GAAG;AAC5B,UAAO,OAAO,WAAW,OAAO,OAAO,SAAS,CAAC,OAAO;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AAAA,EACF,UAAE;AACA,UAAM,UAAU,WAAW;AAAA,EAC7B;AACF;","names":["readFile","join"]}
|
|
1
|
+
{"version":3,"sources":["../src/commands/warehouse-sync.ts","../src/commands/sync-warehouse.ts"],"sourcesContent":["/**\n * Core warehouse sync logic.\n *\n * Pure functions for diffing, grouping, building models, and merging catalogs,\n * plus an async orchestrator (syncWarehouse) that does I/O.\n */\n\nimport { readFile, writeFile, mkdir } from 'fs/promises';\nimport { join } from 'path';\nimport type { Connector } from '@yamchart/query';\nimport type { NormalizedColumnWithTable } from './introspection.js';\nimport { getTablesQuery, getColumnsQuery, getDescribeQuery } from './introspection.js';\nimport type { CatalogModel, CatalogData } from '../dbt/catalog.js';\nimport { generateCatalogMd, generateCatalogJson, generateWarehouseSection } from '../dbt/catalog.js';\n\n// ---------------------------------------------------------------------------\n// Interfaces\n// ---------------------------------------------------------------------------\n\nexport interface SyncState {\n syncedAt: string;\n connection: string;\n schemas: string[];\n skipSamples: boolean;\n tables: Record<string, { columnCount: number; lastAltered?: string }>;\n}\n\nexport interface WarehouseTable {\n qualifiedName: string; // \"SCHEMA.TABLE\"\n schema: string;\n name: string;\n type: string; // \"TABLE\" | \"VIEW\"\n columnCount: number;\n}\n\nexport interface SyncDiff {\n newTables: string[];\n changedTables: string[];\n unchangedTables: string[];\n removedTables: string[];\n}\n\nexport interface SyncWarehouseResult {\n tablesTotal: number;\n tablesNew: number;\n tablesChanged: number;\n tablesUnchanged: number;\n tablesRemoved: number;\n errors: string[];\n durationMs: number;\n}\n\nexport interface SyncWarehouseOptions {\n schemas?: string[];\n database?: string;\n skipSamples?: boolean;\n full?: boolean;\n onProgress?: (table: string, index: number, total: number) => void;\n}\n\n// ---------------------------------------------------------------------------\n// Pure functions\n// ---------------------------------------------------------------------------\n\n/**\n * Group a flat column list into a Map keyed by \"SCHEMA.TABLE\".\n */\nexport function groupColumnsByTable(\n columns: NormalizedColumnWithTable[],\n): Map<string, NormalizedColumnWithTable[]> {\n const map = new Map<string, NormalizedColumnWithTable[]>();\n for (const col of columns) {\n const key = `${col.schema}.${col.table}`;\n const existing = map.get(key);\n if (existing) {\n existing.push(col);\n } else {\n map.set(key, [col]);\n }\n }\n return map;\n}\n\n/**\n * Compare current warehouse tables against previous sync state.\n * All tables are \"new\" when prev is null.\n * Detects changed by column count mismatch.\n * Detects removed tables (in prev but not in current).\n */\nexport function diffSyncState(\n current: WarehouseTable[],\n prev: SyncState | null,\n): SyncDiff {\n const newTables: string[] = [];\n const changedTables: string[] = [];\n const unchangedTables: string[] = [];\n const removedTables: string[] = [];\n\n if (!prev) {\n return {\n newTables: current.map((t) => t.qualifiedName),\n changedTables: [],\n unchangedTables: [],\n removedTables: [],\n };\n }\n\n const currentNames = new Set(current.map((t) => t.qualifiedName));\n\n for (const table of current) {\n const prevEntry = prev.tables[table.qualifiedName];\n if (!prevEntry) {\n newTables.push(table.qualifiedName);\n } else if (prevEntry.columnCount !== table.columnCount) {\n changedTables.push(table.qualifiedName);\n } else {\n unchangedTables.push(table.qualifiedName);\n }\n }\n\n // Detect removed tables\n for (const name of Object.keys(prev.tables)) {\n if (!currentNames.has(name)) {\n removedTables.push(name);\n }\n }\n\n return { newTables, changedTables, unchangedTables, removedTables };\n}\n\n/**\n * Build CatalogModel entries from warehouse metadata.\n */\nexport function buildWarehouseModels(\n tables: WarehouseTable[],\n columnMap: Map<string, NormalizedColumnWithTable[]>,\n samples: Record<string, Record<string, unknown>[]>,\n): CatalogModel[] {\n const now = new Date().toISOString();\n\n return tables.map((table) => {\n const cols = columnMap.get(table.qualifiedName) || [];\n const sampleRows = samples[table.qualifiedName];\n\n return {\n name: table.name,\n path: '',\n description: '',\n table: table.qualifiedName,\n tags: ['warehouse'],\n meta: {},\n columns: cols.map((c) => ({\n name: c.name,\n data_type: c.type,\n description: '',\n hints: [],\n })),\n yamchartModels: [],\n source: 'warehouse' as const,\n tableType: table.type as 'TABLE' | 'VIEW',\n ...(sampleRows ? { sampleRows } : {}),\n lastSynced: now,\n };\n });\n}\n\n/**\n * Merge warehouse models into an existing dbt catalog.\n *\n * 1. If no existing catalog, create new with warehouseModels.\n * 2. Remove stale warehouse entries from existing.\n * 3. For tables in both dbt and warehouse (matched by `table` field, case-insensitive):\n * - dbt entry wins (keep it)\n * - Backfill data_type from warehouse columns where missing\n * - Add any warehouse columns that don't exist in the dbt entry\n * 4. Append remaining warehouse-only models.\n * 5. Return updated CatalogData.\n */\nexport function mergeWithDbtCatalog(\n existingCatalog: CatalogData | null,\n warehouseModels: CatalogModel[],\n): CatalogData {\n if (!existingCatalog) {\n return {\n syncedAt: new Date().toISOString(),\n source: { type: 'warehouse' },\n stats: { modelsIncluded: warehouseModels.length, modelsExcluded: 0 },\n models: warehouseModels,\n };\n }\n\n // Build a lookup of warehouse models by normalized table name\n const warehouseByTable = new Map<string, CatalogModel>();\n for (const wm of warehouseModels) {\n if (wm.table) {\n warehouseByTable.set(wm.table.toUpperCase(), wm);\n }\n }\n\n // Track which warehouse models were consumed by dbt merge\n const consumedWarehouseTables = new Set<string>();\n\n // Process existing catalog: keep non-warehouse entries, merge dbt+warehouse matches\n const mergedModels: CatalogModel[] = [];\n\n for (const model of existingCatalog.models) {\n // Remove stale warehouse entries — they'll be replaced by current warehouseModels\n if (model.source === 'warehouse') {\n continue;\n }\n\n // Check if this dbt model matches a warehouse table\n const tableKey = (model.table || '').toUpperCase();\n const warehouseMatch = tableKey ? warehouseByTable.get(tableKey) : undefined;\n\n if (warehouseMatch) {\n // dbt entry wins — backfill data_type and add missing columns\n const warehouseColMap = new Map<string, { data_type?: string; description: string; hints: string[] }>();\n for (const wc of warehouseMatch.columns) {\n warehouseColMap.set(wc.name.toLowerCase(), wc);\n }\n\n // Backfill data_type on existing dbt columns\n const updatedColumns = model.columns.map((col) => {\n const wc = warehouseColMap.get(col.name.toLowerCase());\n if (wc && !col.data_type && wc.data_type) {\n return { ...col, data_type: wc.data_type };\n }\n return col;\n });\n\n // Add warehouse columns not present in dbt entry\n const existingColNames = new Set(model.columns.map((c) => c.name.toLowerCase()));\n for (const wc of warehouseMatch.columns) {\n if (!existingColNames.has(wc.name.toLowerCase())) {\n updatedColumns.push(wc);\n }\n }\n\n mergedModels.push({ ...model, columns: updatedColumns });\n consumedWarehouseTables.add(tableKey);\n } else {\n mergedModels.push(model);\n }\n }\n\n // Append warehouse-only models (not consumed by dbt merge)\n for (const wm of warehouseModels) {\n const tableKey = (wm.table || '').toUpperCase();\n if (!consumedWarehouseTables.has(tableKey)) {\n mergedModels.push(wm);\n }\n }\n\n return {\n ...existingCatalog,\n syncedAt: new Date().toISOString(),\n models: mergedModels,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Async orchestrator\n// ---------------------------------------------------------------------------\n\n/**\n * Main warehouse sync orchestrator.\n *\n * 1. Fetch tables and columns from the warehouse.\n * 2. Diff against previous sync state.\n * 3. Sample new/changed tables (unless skipSamples).\n * 4. Build warehouse CatalogModel entries.\n * 5. Merge with existing catalog, write files.\n */\nexport async function syncWarehouse(\n projectDir: string,\n connector: Connector,\n connectionType: string,\n connectionName: string,\n options?: SyncWarehouseOptions,\n): Promise<SyncWarehouseResult> {\n const start = performance.now();\n const errors: string[] = [];\n\n // 1. Fetch tables\n const tablesQuery = getTablesQuery(connectionType, {\n database: options?.database,\n });\n const tablesResult = await connector.execute(tablesQuery.sql);\n let normalizedTables = tablesQuery.normalize(tablesResult.rows);\n\n // Filter by schemas if specified\n if (options?.schemas && options.schemas.length > 0) {\n const schemaSet = new Set(options.schemas.map((s) => s.toUpperCase()));\n normalizedTables = normalizedTables.filter((t) => schemaSet.has(t.schema.toUpperCase()));\n }\n\n // 2. Fetch columns\n let allColumns: NormalizedColumnWithTable[] = [];\n\n if (connectionType === 'sqlite') {\n // SQLite: describe each table individually\n for (const table of normalizedTables) {\n try {\n const describeQuery = getDescribeQuery('sqlite', table.name);\n const descResult = await connector.execute(describeQuery.sql);\n const cols = describeQuery.normalize(descResult.rows);\n for (const col of cols) {\n allColumns.push({\n schema: table.schema,\n table: table.name,\n name: col.name,\n type: col.type,\n nullable: col.nullable,\n });\n }\n } catch (err) {\n errors.push(`Failed to describe ${table.name}: ${err instanceof Error ? err.message : String(err)}`);\n }\n }\n } else {\n const columnsQuery = getColumnsQuery(connectionType, {\n database: options?.database,\n });\n const columnsResult = await connector.execute(columnsQuery.sql);\n allColumns = columnsQuery.normalize(columnsResult.rows);\n\n // Filter columns by schemas if specified\n if (options?.schemas && options.schemas.length > 0) {\n const schemaSet = new Set(options.schemas.map((s) => s.toUpperCase()));\n allColumns = allColumns.filter((c) => schemaSet.has(c.schema.toUpperCase()));\n }\n }\n\n // Group columns by table\n const columnMap = groupColumnsByTable(allColumns);\n\n // Build WarehouseTable entries with column counts\n const warehouseTables: WarehouseTable[] = normalizedTables.map((t) => {\n const qualifiedName = t.schema ? `${t.schema}.${t.name}` : t.name;\n return {\n qualifiedName,\n schema: t.schema,\n name: t.name,\n type: t.type,\n columnCount: columnMap.get(qualifiedName)?.length || 0,\n };\n });\n\n // 3. Load previous sync state, compute diff\n let prevState: SyncState | null = null;\n if (!options?.full) {\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), 'utf-8');\n prevState = JSON.parse(raw);\n } catch {\n /* no previous state */\n }\n }\n\n const diff = diffSyncState(warehouseTables, prevState);\n\n // 4. Sample new/changed tables (unless skipSamples)\n const samples: Record<string, Record<string, unknown>[]> = {};\n const tablesToSample = options?.skipSamples\n ? []\n : [...diff.newTables, ...diff.changedTables];\n\n for (let i = 0; i < tablesToSample.length; i++) {\n const tableName = tablesToSample[i]!;\n options?.onProgress?.(tableName, i + 1, tablesToSample.length);\n\n try {\n const sampleSql = `SELECT * FROM ${tableName} LIMIT 3`;\n const samplePromise = connector.execute(sampleSql);\n const timeoutPromise = new Promise<never>((_, reject) =>\n setTimeout(() => reject(new Error(`Timeout sampling ${tableName} (5s)`)), 5000),\n );\n const result = await Promise.race([samplePromise, timeoutPromise]);\n samples[tableName] = result.rows;\n } catch (err) {\n errors.push(`Failed to sample ${tableName}: ${err instanceof Error ? err.message : String(err)}`);\n }\n }\n\n // 5. For unchanged tables: carry forward sampleRows from existing catalog\n let existingCatalog: CatalogData | null = null;\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'catalog.json'), 'utf-8');\n existingCatalog = JSON.parse(raw);\n } catch {\n /* no existing catalog */\n }\n\n if (existingCatalog && !options?.skipSamples) {\n for (const tableName of diff.unchangedTables) {\n const existingModel = existingCatalog.models.find(\n (m) => m.table?.toUpperCase() === tableName.toUpperCase() && m.source === 'warehouse',\n );\n if (existingModel?.sampleRows) {\n samples[tableName] = existingModel.sampleRows;\n }\n }\n }\n\n // 6. Build warehouse CatalogModel entries\n const warehouseModels = buildWarehouseModels(warehouseTables, columnMap, samples);\n\n // 7. Merge with existing catalog, write files\n const merged = mergeWithDbtCatalog(existingCatalog, warehouseModels);\n\n await mkdir(join(projectDir, '.yamchart'), { recursive: true });\n await writeFile(join(projectDir, '.yamchart', 'catalog.json'), generateCatalogJson(merged));\n\n const catalogMd = generateCatalogMd(merged);\n const syncDate = new Date().toISOString().split('T')[0]!;\n const warehouseSection = generateWarehouseSection(merged.models, connectionName, syncDate);\n const fullMd = warehouseSection ? catalogMd + '\\n' + warehouseSection : catalogMd;\n await writeFile(join(projectDir, '.yamchart', 'catalog.md'), fullMd);\n\n // Write sync state\n const syncState: SyncState = {\n syncedAt: new Date().toISOString(),\n connection: connectionName,\n schemas: options?.schemas || [],\n skipSamples: options?.skipSamples || false,\n tables: Object.fromEntries(\n warehouseTables.map((t) => [t.qualifiedName, { columnCount: t.columnCount }]),\n ),\n };\n await writeFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), JSON.stringify(syncState, null, 2));\n\n const durationMs = Math.round(performance.now() - start);\n\n return {\n tablesTotal: warehouseTables.length,\n tablesNew: diff.newTables.length,\n tablesChanged: diff.changedTables.length,\n tablesUnchanged: diff.unchangedTables.length,\n tablesRemoved: diff.removedTables.length,\n errors,\n durationMs,\n };\n}\n","/**\n * CLI handler for `yamchart sync-warehouse`.\n *\n * Thin wrapper: resolves connection, creates connector, delegates to syncWarehouse().\n */\n\nimport * as output from '../utils/output.js';\nimport { resolveConnection, createConnector } from './connection-utils.js';\nimport { syncWarehouse } from './warehouse-sync.js';\nimport { readFile } from 'fs/promises';\nimport { join } from 'path';\nimport type { SyncState } from './warehouse-sync.js';\n\nexport interface SyncWarehouseCliOptions {\n connection?: string;\n schema?: string; // comma-separated → split into array\n database?: string;\n skipSamples?: boolean;\n refresh?: boolean;\n full?: boolean;\n json?: boolean;\n}\n\nexport async function runSyncWarehouse(\n projectDir: string,\n options: SyncWarehouseCliOptions,\n): Promise<void> {\n // Handle --refresh: load saved connection/schemas from previous sync state\n let connectionOverride = options.connection;\n let schemasOverride = options.schema;\n\n if (options.refresh) {\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), 'utf-8');\n const prevState: SyncState = JSON.parse(raw);\n connectionOverride = connectionOverride || prevState.connection;\n if (!schemasOverride && prevState.schemas.length > 0) {\n schemasOverride = prevState.schemas.join(',');\n }\n if (!options.json) {\n output.info(`Re-syncing from ${prevState.connection} (${prevState.schemas.join(', ') || 'all schemas'})`);\n }\n } catch {\n output.error('No previous sync state found. Run sync-warehouse without --refresh first.');\n process.exit(1);\n }\n }\n\n // Resolve connection\n const connection = await resolveConnection(projectDir, connectionOverride);\n const connector = createConnector(connection, projectDir);\n\n const schemas = schemasOverride\n ? schemasOverride.split(',').map((s) => s.trim())\n : undefined;\n\n const spin = options.json ? null : output.spinner(`Connecting to ${connection.name} ...`);\n\n try {\n await connector.connect();\n\n if (spin) {\n spin.text = 'Fetching table metadata...';\n }\n\n const result = await syncWarehouse(\n projectDir,\n connector,\n connection.type,\n connection.name,\n {\n schemas,\n database: options.database,\n skipSamples: options.skipSamples,\n full: options.full,\n onProgress: spin\n ? (table, index, total) => {\n spin.text = `Sampling tables... ${index}/${total} (${table})`;\n }\n : undefined,\n },\n );\n\n if (spin) {\n spin.stop();\n }\n\n if (options.json) {\n console.log(JSON.stringify(result, null, 2));\n } else {\n const parts = [\n `${result.tablesNew} new`,\n `${result.tablesChanged} changed`,\n `${result.tablesUnchanged} unchanged`,\n ];\n if (result.tablesRemoved > 0) {\n parts.push(`${result.tablesRemoved} removed`);\n }\n output.success(\n `Synced ${result.tablesTotal} tables (${parts.join(', ')}) in ${(result.durationMs / 1000).toFixed(1)}s`,\n );\n output.detail('Written to .yamchart/catalog.json');\n\n if (result.errors.length > 0) {\n output.warning(`${result.errors.length} table(s) had sampling errors:`);\n for (const err of result.errors.slice(0, 5)) {\n output.detail(err);\n }\n if (result.errors.length > 5) {\n output.detail(`... and ${result.errors.length - 5} more`);\n }\n }\n }\n } finally {\n await connector.disconnect();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA,SAAS,UAAU,WAAW,aAAa;AAC3C,SAAS,YAAY;AA2Dd,SAAS,oBACd,SAC0C;AAC1C,QAAM,MAAM,oBAAI,IAAyC;AACzD,aAAW,OAAO,SAAS;AACzB,UAAM,MAAM,GAAG,IAAI,MAAM,IAAI,IAAI,KAAK;AACtC,UAAM,WAAW,IAAI,IAAI,GAAG;AAC5B,QAAI,UAAU;AACZ,eAAS,KAAK,GAAG;AAAA,IACnB,OAAO;AACL,UAAI,IAAI,KAAK,CAAC,GAAG,CAAC;AAAA,IACpB;AAAA,EACF;AACA,SAAO;AACT;AAQO,SAAS,cACd,SACA,MACU;AACV,QAAM,YAAsB,CAAC;AAC7B,QAAM,gBAA0B,CAAC;AACjC,QAAM,kBAA4B,CAAC;AACnC,QAAM,gBAA0B,CAAC;AAEjC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,MACL,WAAW,QAAQ,IAAI,CAAC,MAAM,EAAE,aAAa;AAAA,MAC7C,eAAe,CAAC;AAAA,MAChB,iBAAiB,CAAC;AAAA,MAClB,eAAe,CAAC;AAAA,IAClB;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC;AAEhE,aAAW,SAAS,SAAS;AAC3B,UAAM,YAAY,KAAK,OAAO,MAAM,aAAa;AACjD,QAAI,CAAC,WAAW;AACd,gBAAU,KAAK,MAAM,aAAa;AAAA,IACpC,WAAW,UAAU,gBAAgB,MAAM,aAAa;AACtD,oBAAc,KAAK,MAAM,aAAa;AAAA,IACxC,OAAO;AACL,sBAAgB,KAAK,MAAM,aAAa;AAAA,IAC1C;AAAA,EACF;AAGA,aAAW,QAAQ,OAAO,KAAK,KAAK,MAAM,GAAG;AAC3C,QAAI,CAAC,aAAa,IAAI,IAAI,GAAG;AAC3B,oBAAc,KAAK,IAAI;AAAA,IACzB;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,eAAe,iBAAiB,cAAc;AACpE;AAKO,SAAS,qBACd,QACA,WACA,SACgB;AAChB,QAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AAEnC,SAAO,OAAO,IAAI,CAAC,UAAU;AAC3B,UAAM,OAAO,UAAU,IAAI,MAAM,aAAa,KAAK,CAAC;AACpD,UAAM,aAAa,QAAQ,MAAM,aAAa;AAE9C,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,MAAM;AAAA,MACN,aAAa;AAAA,MACb,OAAO,MAAM;AAAA,MACb,MAAM,CAAC,WAAW;AAAA,MAClB,MAAM,CAAC;AAAA,MACP,SAAS,KAAK,IAAI,CAAC,OAAO;AAAA,QACxB,MAAM,EAAE;AAAA,QACR,WAAW,EAAE;AAAA,QACb,aAAa;AAAA,QACb,OAAO,CAAC;AAAA,MACV,EAAE;AAAA,MACF,gBAAgB,CAAC;AAAA,MACjB,QAAQ;AAAA,MACR,WAAW,MAAM;AAAA,MACjB,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,MACnC,YAAY;AAAA,IACd;AAAA,EACF,CAAC;AACH;AAcO,SAAS,oBACd,iBACA,iBACa;AACb,MAAI,CAAC,iBAAiB;AACpB,WAAO;AAAA,MACL,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,MACjC,QAAQ,EAAE,MAAM,YAAY;AAAA,MAC5B,OAAO,EAAE,gBAAgB,gBAAgB,QAAQ,gBAAgB,EAAE;AAAA,MACnE,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,QAAM,mBAAmB,oBAAI,IAA0B;AACvD,aAAW,MAAM,iBAAiB;AAChC,QAAI,GAAG,OAAO;AACZ,uBAAiB,IAAI,GAAG,MAAM,YAAY,GAAG,EAAE;AAAA,IACjD;AAAA,EACF;AAGA,QAAM,0BAA0B,oBAAI,IAAY;AAGhD,QAAM,eAA+B,CAAC;AAEtC,aAAW,SAAS,gBAAgB,QAAQ;AAE1C,QAAI,MAAM,WAAW,aAAa;AAChC;AAAA,IACF;AAGA,UAAM,YAAY,MAAM,SAAS,IAAI,YAAY;AACjD,UAAM,iBAAiB,WAAW,iBAAiB,IAAI,QAAQ,IAAI;AAEnE,QAAI,gBAAgB;AAElB,YAAM,kBAAkB,oBAAI,IAA0E;AACtG,iBAAW,MAAM,eAAe,SAAS;AACvC,wBAAgB,IAAI,GAAG,KAAK,YAAY,GAAG,EAAE;AAAA,MAC/C;AAGA,YAAM,iBAAiB,MAAM,QAAQ,IAAI,CAAC,QAAQ;AAChD,cAAM,KAAK,gBAAgB,IAAI,IAAI,KAAK,YAAY,CAAC;AACrD,YAAI,MAAM,CAAC,IAAI,aAAa,GAAG,WAAW;AACxC,iBAAO,EAAE,GAAG,KAAK,WAAW,GAAG,UAAU;AAAA,QAC3C;AACA,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,mBAAmB,IAAI,IAAI,MAAM,QAAQ,IAAI,CAAC,MAAM,EAAE,KAAK,YAAY,CAAC,CAAC;AAC/E,iBAAW,MAAM,eAAe,SAAS;AACvC,YAAI,CAAC,iBAAiB,IAAI,GAAG,KAAK,YAAY,CAAC,GAAG;AAChD,yBAAe,KAAK,EAAE;AAAA,QACxB;AAAA,MACF;AAEA,mBAAa,KAAK,EAAE,GAAG,OAAO,SAAS,eAAe,CAAC;AACvD,8BAAwB,IAAI,QAAQ;AAAA,IACtC,OAAO;AACL,mBAAa,KAAK,KAAK;AAAA,IACzB;AAAA,EACF;AAGA,aAAW,MAAM,iBAAiB;AAChC,UAAM,YAAY,GAAG,SAAS,IAAI,YAAY;AAC9C,QAAI,CAAC,wBAAwB,IAAI,QAAQ,GAAG;AAC1C,mBAAa,KAAK,EAAE;AAAA,IACtB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG;AAAA,IACH,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,IACjC,QAAQ;AAAA,EACV;AACF;AAeA,eAAsB,cACpB,YACA,WACA,gBACA,gBACA,SAC8B;AAC9B,QAAM,QAAQ,YAAY,IAAI;AAC9B,QAAM,SAAmB,CAAC;AAG1B,QAAM,cAAc,eAAe,gBAAgB;AAAA,IACjD,UAAU,SAAS;AAAA,EACrB,CAAC;AACD,QAAM,eAAe,MAAM,UAAU,QAAQ,YAAY,GAAG;AAC5D,MAAI,mBAAmB,YAAY,UAAU,aAAa,IAAI;AAG9D,MAAI,SAAS,WAAW,QAAQ,QAAQ,SAAS,GAAG;AAClD,UAAM,YAAY,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACrE,uBAAmB,iBAAiB,OAAO,CAAC,MAAM,UAAU,IAAI,EAAE,OAAO,YAAY,CAAC,CAAC;AAAA,EACzF;AAGA,MAAI,aAA0C,CAAC;AAE/C,MAAI,mBAAmB,UAAU;AAE/B,eAAW,SAAS,kBAAkB;AACpC,UAAI;AACF,cAAM,gBAAgB,iBAAiB,UAAU,MAAM,IAAI;AAC3D,cAAM,aAAa,MAAM,UAAU,QAAQ,cAAc,GAAG;AAC5D,cAAM,OAAO,cAAc,UAAU,WAAW,IAAI;AACpD,mBAAW,OAAO,MAAM;AACtB,qBAAW,KAAK;AAAA,YACd,QAAQ,MAAM;AAAA,YACd,OAAO,MAAM;AAAA,YACb,MAAM,IAAI;AAAA,YACV,MAAM,IAAI;AAAA,YACV,UAAU,IAAI;AAAA,UAChB,CAAC;AAAA,QACH;AAAA,MACF,SAAS,KAAK;AACZ,eAAO,KAAK,sBAAsB,MAAM,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,MACrG;AAAA,IACF;AAAA,EACF,OAAO;AACL,UAAM,eAAe,gBAAgB,gBAAgB;AAAA,MACnD,UAAU,SAAS;AAAA,IACrB,CAAC;AACD,UAAM,gBAAgB,MAAM,UAAU,QAAQ,aAAa,GAAG;AAC9D,iBAAa,aAAa,UAAU,cAAc,IAAI;AAGtD,QAAI,SAAS,WAAW,QAAQ,QAAQ,SAAS,GAAG;AAClD,YAAM,YAAY,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACrE,mBAAa,WAAW,OAAO,CAAC,MAAM,UAAU,IAAI,EAAE,OAAO,YAAY,CAAC,CAAC;AAAA,IAC7E;AAAA,EACF;AAGA,QAAM,YAAY,oBAAoB,UAAU;AAGhD,QAAM,kBAAoC,iBAAiB,IAAI,CAAC,MAAM;AACpE,UAAM,gBAAgB,EAAE,SAAS,GAAG,EAAE,MAAM,IAAI,EAAE,IAAI,KAAK,EAAE;AAC7D,WAAO;AAAA,MACL;AAAA,MACA,QAAQ,EAAE;AAAA,MACV,MAAM,EAAE;AAAA,MACR,MAAM,EAAE;AAAA,MACR,aAAa,UAAU,IAAI,aAAa,GAAG,UAAU;AAAA,IACvD;AAAA,EACF,CAAC;AAGD,MAAI,YAA8B;AAClC,MAAI,CAAC,SAAS,MAAM;AAClB,QAAI;AACF,YAAM,MAAM,MAAM,SAAS,KAAK,YAAY,aAAa,qBAAqB,GAAG,OAAO;AACxF,kBAAY,KAAK,MAAM,GAAG;AAAA,IAC5B,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,QAAM,OAAO,cAAc,iBAAiB,SAAS;AAGrD,QAAM,UAAqD,CAAC;AAC5D,QAAM,iBAAiB,SAAS,cAC5B,CAAC,IACD,CAAC,GAAG,KAAK,WAAW,GAAG,KAAK,aAAa;AAE7C,WAAS,IAAI,GAAG,IAAI,eAAe,QAAQ,KAAK;AAC9C,UAAM,YAAY,eAAe,CAAC;AAClC,aAAS,aAAa,WAAW,IAAI,GAAG,eAAe,MAAM;AAE7D,QAAI;AACF,YAAM,YAAY,iBAAiB,SAAS;AAC5C,YAAM,gBAAgB,UAAU,QAAQ,SAAS;AACjD,YAAM,iBAAiB,IAAI;AAAA,QAAe,CAAC,GAAG,WAC5C,WAAW,MAAM,OAAO,IAAI,MAAM,oBAAoB,SAAS,OAAO,CAAC,GAAG,GAAI;AAAA,MAChF;AACA,YAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,eAAe,cAAc,CAAC;AACjE,cAAQ,SAAS,IAAI,OAAO;AAAA,IAC9B,SAAS,KAAK;AACZ,aAAO,KAAK,oBAAoB,SAAS,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,IAClG;AAAA,EACF;AAGA,MAAI,kBAAsC;AAC1C,MAAI;AACF,UAAM,MAAM,MAAM,SAAS,KAAK,YAAY,aAAa,cAAc,GAAG,OAAO;AACjF,sBAAkB,KAAK,MAAM,GAAG;AAAA,EAClC,QAAQ;AAAA,EAER;AAEA,MAAI,mBAAmB,CAAC,SAAS,aAAa;AAC5C,eAAW,aAAa,KAAK,iBAAiB;AAC5C,YAAM,gBAAgB,gBAAgB,OAAO;AAAA,QAC3C,CAAC,MAAM,EAAE,OAAO,YAAY,MAAM,UAAU,YAAY,KAAK,EAAE,WAAW;AAAA,MAC5E;AACA,UAAI,eAAe,YAAY;AAC7B,gBAAQ,SAAS,IAAI,cAAc;AAAA,MACrC;AAAA,IACF;AAAA,EACF;AAGA,QAAM,kBAAkB,qBAAqB,iBAAiB,WAAW,OAAO;AAGhF,QAAM,SAAS,oBAAoB,iBAAiB,eAAe;AAEnE,QAAM,MAAM,KAAK,YAAY,WAAW,GAAG,EAAE,WAAW,KAAK,CAAC;AAC9D,QAAM,UAAU,KAAK,YAAY,aAAa,cAAc,GAAG,oBAAoB,MAAM,CAAC;AAE1F,QAAM,YAAY,kBAAkB,MAAM;AAC1C,QAAM,YAAW,oBAAI,KAAK,GAAE,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC;AACtD,QAAM,mBAAmB,yBAAyB,OAAO,QAAQ,gBAAgB,QAAQ;AACzF,QAAM,SAAS,mBAAmB,YAAY,OAAO,mBAAmB;AACxE,QAAM,UAAU,KAAK,YAAY,aAAa,YAAY,GAAG,MAAM;AAGnE,QAAM,YAAuB;AAAA,IAC3B,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,IACjC,YAAY;AAAA,IACZ,SAAS,SAAS,WAAW,CAAC;AAAA,IAC9B,aAAa,SAAS,eAAe;AAAA,IACrC,QAAQ,OAAO;AAAA,MACb,gBAAgB,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;AAAA,IAC9E;AAAA,EACF;AACA,QAAM,UAAU,KAAK,YAAY,aAAa,qBAAqB,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;AAExG,QAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AAEvD,SAAO;AAAA,IACL,aAAa,gBAAgB;AAAA,IAC7B,WAAW,KAAK,UAAU;AAAA,IAC1B,eAAe,KAAK,cAAc;AAAA,IAClC,iBAAiB,KAAK,gBAAgB;AAAA,IACtC,eAAe,KAAK,cAAc;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;;;AClbA,SAAS,YAAAA,iBAAgB;AACzB,SAAS,QAAAC,aAAY;AAarB,eAAsB,iBACpB,YACA,SACe;AAEf,MAAI,qBAAqB,QAAQ;AACjC,MAAI,kBAAkB,QAAQ;AAE9B,MAAI,QAAQ,SAAS;AACnB,QAAI;AACF,YAAM,MAAM,MAAMD,UAASC,MAAK,YAAY,aAAa,qBAAqB,GAAG,OAAO;AACxF,YAAM,YAAuB,KAAK,MAAM,GAAG;AAC3C,2BAAqB,sBAAsB,UAAU;AACrD,UAAI,CAAC,mBAAmB,UAAU,QAAQ,SAAS,GAAG;AACpD,0BAAkB,UAAU,QAAQ,KAAK,GAAG;AAAA,MAC9C;AACA,UAAI,CAAC,QAAQ,MAAM;AACjB,QAAO,KAAK,mBAAmB,UAAU,UAAU,KAAK,UAAU,QAAQ,KAAK,IAAI,KAAK,aAAa,GAAG;AAAA,MAC1G;AAAA,IACF,QAAQ;AACN,MAAO,MAAM,2EAA2E;AACxF,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AAGA,QAAM,aAAa,MAAM,kBAAkB,YAAY,kBAAkB;AACzE,QAAM,YAAY,gBAAgB,YAAY,UAAU;AAExD,QAAM,UAAU,kBACZ,gBAAgB,MAAM,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAC9C;AAEJ,QAAM,OAAO,QAAQ,OAAO,OAAc,QAAQ,iBAAiB,WAAW,IAAI,MAAM;AAExF,MAAI;AACF,UAAM,UAAU,QAAQ;AAExB,QAAI,MAAM;AACR,WAAK,OAAO;AAAA,IACd;AAEA,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,WAAW;AAAA,MACX;AAAA,QACE;AAAA,QACA,UAAU,QAAQ;AAAA,QAClB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,QACd,YAAY,OACR,CAAC,OAAO,OAAO,UAAU;AACvB,eAAK,OAAO,sBAAsB,KAAK,IAAI,KAAK,KAAK,KAAK;AAAA,QAC5D,IACA;AAAA,MACN;AAAA,IACF;AAEA,QAAI,MAAM;AACR,WAAK,KAAK;AAAA,IACZ;AAEA,QAAI,QAAQ,MAAM;AAChB,cAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,IAC7C,OAAO;AACL,YAAM,QAAQ;AAAA,QACZ,GAAG,OAAO,SAAS;AAAA,QACnB,GAAG,OAAO,aAAa;AAAA,QACvB,GAAG,OAAO,eAAe;AAAA,MAC3B;AACA,UAAI,OAAO,gBAAgB,GAAG;AAC5B,cAAM,KAAK,GAAG,OAAO,aAAa,UAAU;AAAA,MAC9C;AACA,MAAO;AAAA,QACL,UAAU,OAAO,WAAW,YAAY,MAAM,KAAK,IAAI,CAAC,SAAS,OAAO,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,MACvG;AACA,MAAO,OAAO,mCAAmC;AAEjD,UAAI,OAAO,OAAO,SAAS,GAAG;AAC5B,QAAO,QAAQ,GAAG,OAAO,OAAO,MAAM,gCAAgC;AACtE,mBAAW,OAAO,OAAO,OAAO,MAAM,GAAG,CAAC,GAAG;AAC3C,UAAO,OAAO,GAAG;AAAA,QACnB;AACA,YAAI,OAAO,OAAO,SAAS,GAAG;AAC5B,UAAO,OAAO,WAAW,OAAO,OAAO,SAAS,CAAC,OAAO;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AAAA,EACF,UAAE;AACA,UAAM,UAAU,WAAW;AAAA,EAC7B;AACF;","names":["readFile","join"]}
|
|
@@ -4,9 +4,10 @@ import {
|
|
|
4
4
|
import {
|
|
5
5
|
createConnector,
|
|
6
6
|
resolveConnection
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
7
|
+
} from "./chunk-OTAUP5RC.js";
|
|
8
|
+
import "./chunk-X6LQGWUX.js";
|
|
9
|
+
import "./chunk-JYQKDWLG.js";
|
|
10
|
+
import "./chunk-UND73EOB.js";
|
|
10
11
|
import "./chunk-DGUM43GV.js";
|
|
11
12
|
|
|
12
13
|
// src/commands/tables.ts
|
|
@@ -36,4 +37,4 @@ async function listTables(projectDir, options) {
|
|
|
36
37
|
export {
|
|
37
38
|
listTables
|
|
38
39
|
};
|
|
39
|
-
//# sourceMappingURL=tables-
|
|
40
|
+
//# sourceMappingURL=tables-UOO342TA.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/commands/tables.ts"],"sourcesContent":["import { resolveConnection, createConnector } from './connection-utils.js';\nimport { getTablesQuery, type NormalizedTable } from './introspection.js';\n\nexport interface TablesOptions {\n connection?: string;\n schema?: string;\n database?: string;\n json?: boolean;\n}\n\nexport interface TablesResult {\n tables: NormalizedTable[];\n connectionName: string;\n connectionType: string;\n durationMs: number;\n}\n\n/**\n * List all tables and views for a connection.\n *\n * Resolves the connection from the project directory, connects,\n * runs the introspection query, normalizes the results, and disconnects.\n */\nexport async function listTables(\n projectDir: string,\n options: TablesOptions,\n): Promise<TablesResult> {\n const connection = await resolveConnection(projectDir, options.connection);\n const connector = createConnector(connection, projectDir);\n\n const start = performance.now();\n\n try {\n await connector.connect();\n\n const query = getTablesQuery(connection.type, {\n schema: options.schema,\n database: options.database,\n });\n\n const result = await connector.execute(query.sql);\n const tables = query.normalize(result.rows);\n\n const durationMs = Math.round((performance.now() - start) * 100) / 100;\n\n return {\n tables,\n connectionName: connection.name,\n connectionType: connection.type,\n durationMs,\n };\n } finally {\n await connector.disconnect();\n }\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../src/commands/tables.ts"],"sourcesContent":["import { resolveConnection, createConnector } from './connection-utils.js';\nimport { getTablesQuery, type NormalizedTable } from './introspection.js';\n\nexport interface TablesOptions {\n connection?: string;\n schema?: string;\n database?: string;\n json?: boolean;\n}\n\nexport interface TablesResult {\n tables: NormalizedTable[];\n connectionName: string;\n connectionType: string;\n durationMs: number;\n}\n\n/**\n * List all tables and views for a connection.\n *\n * Resolves the connection from the project directory, connects,\n * runs the introspection query, normalizes the results, and disconnects.\n */\nexport async function listTables(\n projectDir: string,\n options: TablesOptions,\n): Promise<TablesResult> {\n const connection = await resolveConnection(projectDir, options.connection);\n const connector = createConnector(connection, projectDir);\n\n const start = performance.now();\n\n try {\n await connector.connect();\n\n const query = getTablesQuery(connection.type, {\n schema: options.schema,\n database: options.database,\n });\n\n const result = await connector.execute(query.sql);\n const tables = query.normalize(result.rows);\n\n const durationMs = Math.round((performance.now() - start) * 100) / 100;\n\n return {\n tables,\n connectionName: connection.name,\n connectionType: connection.type,\n durationMs,\n };\n } finally {\n await connector.disconnect();\n }\n}\n"],"mappings":";;;;;;;;;;;;;AAuBA,eAAsB,WACpB,YACA,SACuB;AACvB,QAAM,aAAa,MAAM,kBAAkB,YAAY,QAAQ,UAAU;AACzE,QAAM,YAAY,gBAAgB,YAAY,UAAU;AAExD,QAAM,QAAQ,YAAY,IAAI;AAE9B,MAAI;AACF,UAAM,UAAU,QAAQ;AAExB,UAAM,QAAQ,eAAe,WAAW,MAAM;AAAA,MAC5C,QAAQ,QAAQ;AAAA,MAChB,UAAU,QAAQ;AAAA,IACpB,CAAC;AAED,UAAM,SAAS,MAAM,UAAU,QAAQ,MAAM,GAAG;AAChD,UAAM,SAAS,MAAM,UAAU,OAAO,IAAI;AAE1C,UAAM,aAAa,KAAK,OAAO,YAAY,IAAI,IAAI,SAAS,GAAG,IAAI;AAEnE,WAAO;AAAA,MACL;AAAA,MACA,gBAAgB,WAAW;AAAA,MAC3B,gBAAgB,WAAW;AAAA,MAC3B;AAAA,IACF;AAAA,EACF,UAAE;AACA,UAAM,UAAU,WAAW;AAAA,EAC7B;AACF;","names":[]}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: databricks
|
|
3
|
+
description: Databricks database exploration and model writing. Use when user is working with a Databricks connection or asks about Databricks/Unity Catalog tables, queries, or models.
|
|
4
|
+
argument-hint: "[optional: table name or question]"
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Databricks Database Explorer
|
|
8
|
+
|
|
9
|
+
Help the user explore their Databricks database and write yamchart SQL models.
|
|
10
|
+
|
|
11
|
+
## Discovery Workflow
|
|
12
|
+
|
|
13
|
+
1. **List tables:** `yamchart tables --connection <name>` or filter with `--schema <schema>` / `--database <catalog>`
|
|
14
|
+
2. **Describe columns:** `yamchart describe catalog.schema.table --connection <name>`
|
|
15
|
+
3. **Sample rows:** `yamchart sample catalog.schema.table -l 10 --connection <name>` — Show sample rows (default: 5). Accepts dbt model names.
|
|
16
|
+
4. **Search:** `yamchart search <keyword> --connection <name>` — Find tables and columns matching keyword.
|
|
17
|
+
5. **Ad-hoc query:** `yamchart query "SELECT * FROM catalog.schema.table LIMIT 10" --connection <name>`
|
|
18
|
+
6. **Write model** in `models/*.sql` using what you learned
|
|
19
|
+
|
|
20
|
+
If the user hasn't specified a connection, check `yamchart.yaml` for the default connection or list connections in `connections/*.yaml`.
|
|
21
|
+
|
|
22
|
+
## Databricks-Specific Knowledge
|
|
23
|
+
|
|
24
|
+
**Naming (Unity Catalog):**
|
|
25
|
+
- Three-level naming: `catalog.schema.table`
|
|
26
|
+
- Default catalog can be set at workspace level
|
|
27
|
+
- Common structure: `prod.gold.fact_orders`, `dev.staging.stg_events`
|
|
28
|
+
- dbt model names often don't match materialized paths
|
|
29
|
+
|
|
30
|
+
**Delta Lake:**
|
|
31
|
+
- All tables are Delta format by default
|
|
32
|
+
- Time travel: `SELECT * FROM table VERSION AS OF 5` or `TIMESTAMP AS OF '2025-01-01'`
|
|
33
|
+
- `DESCRIBE HISTORY table` to see version history
|
|
34
|
+
- `OPTIMIZE table` and `VACUUM table` for maintenance
|
|
35
|
+
|
|
36
|
+
**Type system:**
|
|
37
|
+
- Spark SQL types: `STRING`, `INT`, `BIGINT`, `DOUBLE`, `DECIMAL(p,s)`
|
|
38
|
+
- `TIMESTAMP` (with timezone), `DATE`
|
|
39
|
+
- `ARRAY<type>`, `MAP<key,value>`, `STRUCT<field:type,...>` — complex types are common
|
|
40
|
+
- `BINARY`, `BOOLEAN`, `INTERVAL`
|
|
41
|
+
|
|
42
|
+
**Date/time:**
|
|
43
|
+
- `date_trunc('month', date_col)`, `date_add(date_col, 7)`
|
|
44
|
+
- `datediff(end, start)`, `months_between(end, start)`
|
|
45
|
+
- `current_date()`, `current_timestamp()`
|
|
46
|
+
- `to_date(str, 'yyyy-MM-dd')`, `date_format(date_col, 'yyyy-MM')`
|
|
47
|
+
|
|
48
|
+
**Complex types:**
|
|
49
|
+
- Array access: `col[0]`, `size(col)`
|
|
50
|
+
- `EXPLODE(array_col)` to unnest arrays (similar to Snowflake's FLATTEN)
|
|
51
|
+
- `LATERAL VIEW EXPLODE(array_col) t AS item`
|
|
52
|
+
- Struct access: `col.field_name`
|
|
53
|
+
- Map access: `col['key']`
|
|
54
|
+
|
|
55
|
+
**Useful commands via `yamchart query`:**
|
|
56
|
+
- `SHOW CATALOGS` — list available catalogs
|
|
57
|
+
- `SHOW SCHEMAS IN catalog` — list schemas
|
|
58
|
+
- `SHOW TABLES IN catalog.schema` — list tables
|
|
59
|
+
- `DESCRIBE EXTENDED catalog.schema.table` — detailed table info
|
|
60
|
+
- `SHOW CREATE TABLE catalog.schema.table` — full DDL
|
|
61
|
+
|
|
62
|
+
**Common patterns in yamchart models:**
|
|
63
|
+
```sql
|
|
64
|
+
-- @name: event_metrics
|
|
65
|
+
-- @param start_date: date = 2025-01-01
|
|
66
|
+
-- @param end_date: date = 2025-12-31
|
|
67
|
+
SELECT
|
|
68
|
+
date_trunc('month', event_timestamp) AS period,
|
|
69
|
+
event_type,
|
|
70
|
+
COUNT(*) AS event_count,
|
|
71
|
+
COUNT(DISTINCT user_id) AS unique_users
|
|
72
|
+
FROM prod.gold.events
|
|
73
|
+
WHERE event_timestamp BETWEEN '{{ start_date }}' AND '{{ end_date }}'
|
|
74
|
+
GROUP BY 1, 2
|
|
75
|
+
ORDER BY 1, 2
|
|
76
|
+
```
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: dbt-advisor
|
|
3
|
+
description: AI-powered dbt model advisor. Use when the user wants help designing dbt models informed by their yamchart BI layer, or asks about improving their dbt project structure.
|
|
4
|
+
argument-hint: "[optional: question or 'audit']"
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# dbt Advisor
|
|
8
|
+
|
|
9
|
+
Help the user design and write dbt models informed by their yamchart BI layer.
|
|
10
|
+
|
|
11
|
+
## Setup
|
|
12
|
+
|
|
13
|
+
1. **Find dbt project path:** Read `.yamchart/dbt-source.yaml` for the `path` field. If not found, ask the user where their dbt project is.
|
|
14
|
+
2. **Load yamchart context:** Read files in `models/`, `charts/`, `dashboards/` to understand what the BI layer needs.
|
|
15
|
+
3. **Load dbt context:** Read `dbt_project.yml` and scan the dbt project's `models/` directory to understand existing structure, conventions, and naming patterns.
|
|
16
|
+
4. **Load catalog (cached metadata):** Read `.yamchart/catalog.json` for detailed warehouse metadata (tables, columns, types, sample rows) — this is the primary data source for table lookups. Also read `.yamchart/catalog.md` for a human-readable summary.
|
|
17
|
+
5. **Check catalog freshness:** If `.yamchart/catalog.json` doesn't exist or looks stale, suggest running `yamchart sync-warehouse` to populate it.
|
|
18
|
+
6. **Load knowledge:** Read the bundled dbt reference docs from `packages/advisor/src/knowledge/` for dbt expertise.
|
|
19
|
+
|
|
20
|
+
## Convention Detection
|
|
21
|
+
|
|
22
|
+
Before suggesting new models, analyze the existing dbt project to detect:
|
|
23
|
+
|
|
24
|
+
- **Folder structure:** What layers exist? (staging, intermediate, marts, etc.)
|
|
25
|
+
- **Naming prefixes:** What prefixes are used? (stg_, int_, fct_, dim_, etc.)
|
|
26
|
+
- **Materializations:** What materialization does each layer typically use?
|
|
27
|
+
- **schema.yml pattern:** Per-folder or single-file?
|
|
28
|
+
|
|
29
|
+
Match these conventions in all suggestions.
|
|
30
|
+
|
|
31
|
+
## Data Discovery
|
|
32
|
+
|
|
33
|
+
Use these tools in order of preference (fastest first):
|
|
34
|
+
|
|
35
|
+
### 1. Cached catalog (instant — prefer this)
|
|
36
|
+
- **`.yamchart/catalog.json`** — Full table/column metadata with types and sample rows. Use this for column lookups, type checking, and understanding table structure. No live query needed.
|
|
37
|
+
- **`.yamchart/catalog.md`** — Grouped summary of warehouse tables, dbt models, and sources. Good for quick overview.
|
|
38
|
+
|
|
39
|
+
### 2. CLI search (fast, targeted)
|
|
40
|
+
- **`yamchart search <keyword>`** — Find tables and columns matching a keyword across the entire warehouse. Use when you need to discover tables related to a concept (e.g., `yamchart search revenue`).
|
|
41
|
+
|
|
42
|
+
### 3. CLI introspection (live queries)
|
|
43
|
+
- **`yamchart tables`** — List all tables/views in the connected warehouse
|
|
44
|
+
- **`yamchart tables --schema <name>`** — Filter to a specific schema
|
|
45
|
+
- **`yamchart describe <table>`** — Show columns and types for a specific table (accepts dbt model names)
|
|
46
|
+
- **`yamchart sample <table>`** — Preview sample rows (default: 5, accepts dbt model names)
|
|
47
|
+
- **`yamchart query "SELECT ..."`** — Run ad-hoc SQL for deeper exploration
|
|
48
|
+
|
|
49
|
+
Use live queries only when the catalog is missing or you need data that isn't cached (e.g., checking current row counts, verifying recent schema changes).
|
|
50
|
+
|
|
51
|
+
### 4. Lineage
|
|
52
|
+
- **`yamchart lineage <model>`** — Show upstream dependency tree for a model. Use this to understand what a model depends on before proposing changes, and to avoid breaking downstream consumers.
|
|
53
|
+
|
|
54
|
+
## Workflow
|
|
55
|
+
|
|
56
|
+
### Interactive Mode
|
|
57
|
+
1. Ask what the user wants to build or improve
|
|
58
|
+
2. Check `.yamchart/catalog.json` for relevant table metadata (prefer cached over live)
|
|
59
|
+
3. Use `yamchart search` to discover related tables if needed
|
|
60
|
+
4. Use `yamchart lineage` to understand dependencies of models you plan to modify
|
|
61
|
+
5. Read existing dbt models to understand current coverage
|
|
62
|
+
6. Compare what exists in dbt vs what yamchart charts need
|
|
63
|
+
7. Propose new models with full SQL, matching project conventions
|
|
64
|
+
8. On confirmation, write the SQL file and update schema.yml
|
|
65
|
+
|
|
66
|
+
### Audit Mode
|
|
67
|
+
If the user says "audit", systematically evaluate:
|
|
68
|
+
1. **Coverage gaps** — yamchart charts referencing tables with no dbt model
|
|
69
|
+
2. **Convention violations** — models not following naming/materialization patterns
|
|
70
|
+
3. **Missing tests** — models without schema.yml entries or tests
|
|
71
|
+
4. **Optimization opportunities** — views that should be tables, missing incremental models
|
|
72
|
+
5. **Redundancy** — duplicate logic across models
|
|
73
|
+
6. **Lineage gaps** — models with undocumented or unclear upstream dependencies
|
|
74
|
+
|
|
75
|
+
Rank suggestions by impact and present the top findings.
|
|
76
|
+
|
|
77
|
+
## Writing Models
|
|
78
|
+
|
|
79
|
+
When writing dbt model files:
|
|
80
|
+
1. Place in the correct folder based on layer (staging → `models/staging/`, marts → `models/marts/`)
|
|
81
|
+
2. Use the project's naming convention (e.g., `stg_source__table`, `fct_entity`)
|
|
82
|
+
3. Add `{{ config(materialized='...') }}` matching the layer's convention
|
|
83
|
+
4. Use `{{ ref('...') }}` for dbt model references
|
|
84
|
+
5. Use `{{ source('...', '...') }}` for raw table references
|
|
85
|
+
6. Include a config block only if materialization differs from dbt default (view)
|
|
86
|
+
7. After writing the SQL, update the corresponding `schema.yml` with column descriptions and tests
|
|
87
|
+
|
|
88
|
+
## dbt Reference
|
|
89
|
+
|
|
90
|
+
Quick reference for common patterns. For deeper details, read files in `packages/advisor/src/knowledge/`:
|
|
91
|
+
|
|
92
|
+
### Materializations
|
|
93
|
+
- `view` — staging layer, light transforms
|
|
94
|
+
- `table` — marts, reporting, pre-aggregated
|
|
95
|
+
- `incremental` — large fact tables, event data
|
|
96
|
+
- `ephemeral` — one-off CTEs, never materialized
|
|
97
|
+
|
|
98
|
+
### Naming
|
|
99
|
+
- `stg_source__table` — staging (1:1 with source)
|
|
100
|
+
- `int_description` — intermediate (joins, pivots)
|
|
101
|
+
- `fct_entity` — fact table (events, transactions)
|
|
102
|
+
- `dim_entity` — dimension table (entities, lookups)
|
|
103
|
+
|
|
104
|
+
### Testing
|
|
105
|
+
- Every model: `unique` + `not_null` on primary key
|
|
106
|
+
- Foreign keys: `relationships` test
|
|
107
|
+
- Enums: `accepted_values` test
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: duckdb
|
|
3
|
+
description: DuckDB database exploration and model writing. Use when user is working with a DuckDB connection or asks about DuckDB tables, queries, or models.
|
|
4
|
+
argument-hint: "[optional: table name or question]"
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# DuckDB Database Explorer
|
|
8
|
+
|
|
9
|
+
Help the user explore their DuckDB database and write yamchart SQL models.
|
|
10
|
+
|
|
11
|
+
## Discovery Workflow
|
|
12
|
+
|
|
13
|
+
1. **List tables:** `yamchart tables`
|
|
14
|
+
2. **Describe columns:** `yamchart describe <table>`
|
|
15
|
+
3. **Sample rows:** `yamchart sample <table> -l 10` — Show sample rows (default: 5). Accepts dbt model names.
|
|
16
|
+
4. **Search:** `yamchart search <keyword>` — Find tables and columns matching keyword.
|
|
17
|
+
5. **Ad-hoc query:** `yamchart query "SELECT * FROM table LIMIT 10"`
|
|
18
|
+
6. **Write model** in `models/*.sql` using what you learned
|
|
19
|
+
|
|
20
|
+
If the user hasn't specified a connection, check `yamchart.yaml` for the default connection. DuckDB is often the default.
|
|
21
|
+
|
|
22
|
+
## DuckDB-Specific Knowledge
|
|
23
|
+
|
|
24
|
+
**Naming:**
|
|
25
|
+
- Case-insensitive identifiers (no need for quoting or aliasing to lowercase)
|
|
26
|
+
- Simple flat namespace — typically no schema prefix needed
|
|
27
|
+
- File-based (`.duckdb`) or `:memory:`, paths relative to project root
|
|
28
|
+
|
|
29
|
+
**File reading (powerful):**
|
|
30
|
+
- CSV: `SELECT * FROM read_csv('path/to/file.csv')`
|
|
31
|
+
- Parquet: `SELECT * FROM read_parquet('data/*.parquet')` (supports globs)
|
|
32
|
+
- JSON: `SELECT * FROM read_json('data.json')`
|
|
33
|
+
- Multiple files: `SELECT * FROM read_parquet('data/year=*/month=*/*.parquet')`
|
|
34
|
+
- Remote: `SELECT * FROM read_parquet('https://example.com/data.parquet')`
|
|
35
|
+
|
|
36
|
+
**Extensions:**
|
|
37
|
+
- `INSTALL httpfs; LOAD httpfs;` for remote file access
|
|
38
|
+
- `INSTALL spatial; LOAD spatial;` for geospatial
|
|
39
|
+
- Extensions can be loaded via `yamchart query "INSTALL extension_name"`
|
|
40
|
+
|
|
41
|
+
**Date/time:**
|
|
42
|
+
- `date_trunc('month', date_col)`, `date_add(date_col, INTERVAL 7 DAY)`
|
|
43
|
+
- `strftime(date_col, '%Y-%m')` for formatting
|
|
44
|
+
- `CURRENT_DATE`, `CURRENT_TIMESTAMP`
|
|
45
|
+
- Interval arithmetic: `date_col + INTERVAL '30 days'`
|
|
46
|
+
|
|
47
|
+
**Unique features:**
|
|
48
|
+
- `DESCRIBE SELECT ...` to see output columns of any query
|
|
49
|
+
- `SUMMARIZE table_name` for quick statistics
|
|
50
|
+
- `EXPLAIN` / `EXPLAIN ANALYZE` for query plans
|
|
51
|
+
- List comprehensions: `list_transform(arr, x -> x * 2)`
|
|
52
|
+
- String slicing: `str[1:3]`
|
|
53
|
+
|
|
54
|
+
**Common patterns in yamchart models:**
|
|
55
|
+
```sql
|
|
56
|
+
-- @name: sales_summary
|
|
57
|
+
-- @param start_date: date = 2025-01-01
|
|
58
|
+
-- @param end_date: date = 2025-12-31
|
|
59
|
+
SELECT
|
|
60
|
+
date_trunc('month', order_date) AS period,
|
|
61
|
+
SUM(amount) AS revenue,
|
|
62
|
+
COUNT(*) AS order_count
|
|
63
|
+
FROM orders
|
|
64
|
+
WHERE order_date BETWEEN '{{ start_date }}' AND '{{ end_date }}'
|
|
65
|
+
GROUP BY 1
|
|
66
|
+
ORDER BY 1
|
|
67
|
+
```
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: mysql
|
|
3
|
+
description: MySQL database exploration and model writing. Use when user is working with a MySQL connection or asks about MySQL tables, queries, or models.
|
|
4
|
+
argument-hint: "[optional: table name or question]"
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# MySQL Database Explorer
|
|
8
|
+
|
|
9
|
+
Help the user explore their MySQL database and write yamchart SQL models.
|
|
10
|
+
|
|
11
|
+
## Discovery Workflow
|
|
12
|
+
|
|
13
|
+
1. **List tables:** `yamchart tables --connection <name>` or filter with `--schema database_name`
|
|
14
|
+
2. **Describe columns:** `yamchart describe database.table --connection <name>`
|
|
15
|
+
3. **Sample rows:** `yamchart sample database.table -l 10 --connection <name>` — Show sample rows (default: 5). Accepts dbt model names.
|
|
16
|
+
4. **Search:** `yamchart search <keyword> --connection <name>` — Find tables and columns matching keyword.
|
|
17
|
+
5. **Ad-hoc query:** `yamchart query "SELECT * FROM database.table LIMIT 10" --connection <name>`
|
|
18
|
+
6. **Write model** in `models/*.sql` using what you learned
|
|
19
|
+
|
|
20
|
+
If the user hasn't specified a connection, check `yamchart.yaml` for the default connection or list connections in `connections/*.yaml`.
|
|
21
|
+
|
|
22
|
+
## MySQL-Specific Knowledge
|
|
23
|
+
|
|
24
|
+
**Naming:**
|
|
25
|
+
- In MySQL, "database" and "schema" are synonymous
|
|
26
|
+
- Two-part naming: `database.table`
|
|
27
|
+
- Backticks for reserved words: `` `order` ``, `` `group` ``
|
|
28
|
+
- Case sensitivity depends on OS (case-insensitive on macOS/Windows, sensitive on Linux)
|
|
29
|
+
|
|
30
|
+
**Type system:**
|
|
31
|
+
- `INT`, `BIGINT`, `TINYINT(1)` (boolean), `DECIMAL(p,s)`
|
|
32
|
+
- `VARCHAR(n)` (max 65535), `TEXT`, `MEDIUMTEXT`, `LONGTEXT`
|
|
33
|
+
- `DATETIME` (no timezone), `TIMESTAMP` (with timezone, auto-converts)
|
|
34
|
+
- `JSON` type (MySQL 5.7+)
|
|
35
|
+
- `ENUM('a','b','c')` — common in MySQL schemas
|
|
36
|
+
|
|
37
|
+
**Date/time:**
|
|
38
|
+
- `DATE_FORMAT(date_col, '%Y-%m')`, `STR_TO_DATE(str, '%Y-%m-%d')`
|
|
39
|
+
- `DATE_ADD(date_col, INTERVAL 7 DAY)`, `DATE_SUB(date_col, INTERVAL 1 MONTH)`
|
|
40
|
+
- `DATEDIFF(end, start)` returns days
|
|
41
|
+
- `NOW()`, `CURDATE()`, `CURRENT_TIMESTAMP`
|
|
42
|
+
- No `date_trunc` — use `DATE_FORMAT`: `DATE_FORMAT(order_date, '%Y-%m-01')` for month truncation
|
|
43
|
+
|
|
44
|
+
**Common patterns:**
|
|
45
|
+
- `GROUP_CONCAT(col SEPARATOR ', ')` for string aggregation
|
|
46
|
+
- `IFNULL(a, b)` (MySQL equivalent of `COALESCE` — though `COALESCE` also works)
|
|
47
|
+
- `FIND_IN_SET('value', csv_column)` for comma-separated columns
|
|
48
|
+
- No window functions before MySQL 8.0 — check version
|
|
49
|
+
- `LIMIT offset, count` syntax: `LIMIT 10, 20` (skip 10, take 20)
|
|
50
|
+
|
|
51
|
+
**JSON access (MySQL 5.7+):**
|
|
52
|
+
- `col->'$.key'` returns JSON value
|
|
53
|
+
- `col->>'$.key'` returns unquoted text
|
|
54
|
+
- `JSON_EXTRACT(col, '$.key')`, `JSON_UNQUOTE()`
|
|
55
|
+
|
|
56
|
+
**Common patterns in yamchart models:**
|
|
57
|
+
```sql
|
|
58
|
+
-- @name: monthly_orders
|
|
59
|
+
-- @param start_date: date = 2025-01-01
|
|
60
|
+
-- @param end_date: date = 2025-12-31
|
|
61
|
+
SELECT
|
|
62
|
+
DATE_FORMAT(order_date, '%Y-%m-01') AS period,
|
|
63
|
+
SUM(amount) AS revenue,
|
|
64
|
+
COUNT(*) AS order_count
|
|
65
|
+
FROM orders
|
|
66
|
+
WHERE order_date BETWEEN '{{ start_date }}' AND '{{ end_date }}'
|
|
67
|
+
GROUP BY DATE_FORMAT(order_date, '%Y-%m-01')
|
|
68
|
+
ORDER BY period
|
|
69
|
+
```
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: postgres
|
|
3
|
+
description: PostgreSQL database exploration and model writing. Use when user is working with a Postgres connection or asks about PostgreSQL tables, queries, or models.
|
|
4
|
+
argument-hint: "[optional: table name or question]"
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# PostgreSQL Database Explorer
|
|
8
|
+
|
|
9
|
+
Help the user explore their PostgreSQL database and write yamchart SQL models.
|
|
10
|
+
|
|
11
|
+
## Discovery Workflow
|
|
12
|
+
|
|
13
|
+
1. **List tables:** `yamchart tables --connection <name>` or filter with `--schema public`
|
|
14
|
+
2. **Describe columns:** `yamchart describe schema.table --connection <name>`
|
|
15
|
+
3. **Sample rows:** `yamchart sample schema.table -l 10 --connection <name>` — Show sample rows (default: 5). Accepts dbt model names.
|
|
16
|
+
4. **Search:** `yamchart search <keyword> --connection <name>` — Find tables and columns matching keyword.
|
|
17
|
+
5. **Ad-hoc query:** `yamchart query "SELECT * FROM schema.table LIMIT 10" --connection <name>`
|
|
18
|
+
6. **Write model** in `models/*.sql` using what you learned
|
|
19
|
+
|
|
20
|
+
If the user hasn't specified a connection, check `yamchart.yaml` for the default connection or list connections in `connections/*.yaml`.
|
|
21
|
+
|
|
22
|
+
## PostgreSQL-Specific Knowledge
|
|
23
|
+
|
|
24
|
+
**Naming:**
|
|
25
|
+
- Schema-qualified: `schema.table` (default schema is `public`)
|
|
26
|
+
- Case-sensitive only when double-quoted — unquoted identifiers fold to lowercase
|
|
27
|
+
- Common schemas: `public`, `analytics`, `staging`, `raw`
|
|
28
|
+
|
|
29
|
+
**Type system:**
|
|
30
|
+
- `text` vs `varchar(n)` — prefer `text` in most cases
|
|
31
|
+
- `timestamp` (no timezone) vs `timestamptz` (with timezone) — most apps use `timestamptz`
|
|
32
|
+
- `numeric` / `decimal` for exact precision, `real` / `double precision` for approximate
|
|
33
|
+
- `jsonb` for JSON data (prefer over `json`)
|
|
34
|
+
- `uuid`, `inet`, `cidr`, `macaddr` — rich built-in types
|
|
35
|
+
|
|
36
|
+
**Date/time:**
|
|
37
|
+
- `date_trunc('month', ts)`, `date_part('year', ts)`
|
|
38
|
+
- `NOW()`, `CURRENT_DATE`, `CURRENT_TIMESTAMP`
|
|
39
|
+
- Interval arithmetic: `ts - INTERVAL '30 days'`, `ts + '1 month'::interval`
|
|
40
|
+
- `generate_series('2025-01-01'::date, '2025-12-31'::date, '1 month')` for date spines
|
|
41
|
+
|
|
42
|
+
**Common patterns:**
|
|
43
|
+
- CTEs: `WITH cte AS (SELECT ...) SELECT * FROM cte`
|
|
44
|
+
- Window functions: `ROW_NUMBER() OVER (PARTITION BY x ORDER BY y)`
|
|
45
|
+
- `COALESCE(a, b)`, `NULLIF(a, b)`
|
|
46
|
+
- `STRING_AGG(col, ', ')` for grouping strings
|
|
47
|
+
- `FILTER (WHERE ...)` clause on aggregates: `COUNT(*) FILTER (WHERE status = 'active')`
|
|
48
|
+
|
|
49
|
+
**JSONB access:**
|
|
50
|
+
- `col->>'key'` extracts text value
|
|
51
|
+
- `col->'nested'->'key'` navigates paths
|
|
52
|
+
- `jsonb_array_elements(col)` to unnest arrays
|
|
53
|
+
|
|
54
|
+
**Common patterns in yamchart models:**
|
|
55
|
+
```sql
|
|
56
|
+
-- @name: user_activity
|
|
57
|
+
-- @param start_date: date = 2025-01-01
|
|
58
|
+
-- @param end_date: date = 2025-12-31
|
|
59
|
+
WITH daily AS (
|
|
60
|
+
SELECT
|
|
61
|
+
date_trunc('day', created_at) AS period,
|
|
62
|
+
COUNT(DISTINCT user_id) AS active_users,
|
|
63
|
+
COUNT(*) AS events
|
|
64
|
+
FROM analytics.events
|
|
65
|
+
WHERE created_at BETWEEN '{{ start_date }}' AND '{{ end_date }}'
|
|
66
|
+
GROUP BY 1
|
|
67
|
+
)
|
|
68
|
+
SELECT * FROM daily ORDER BY period
|
|
69
|
+
```
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: snowflake
|
|
3
|
+
description: Snowflake database exploration and model writing. Use when user is working with a Snowflake connection or asks about Snowflake tables, queries, or models.
|
|
4
|
+
argument-hint: "[optional: table name or question]"
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Snowflake Database Explorer
|
|
8
|
+
|
|
9
|
+
Help the user explore their Snowflake database and write yamchart SQL models.
|
|
10
|
+
|
|
11
|
+
## Discovery Workflow
|
|
12
|
+
|
|
13
|
+
1. **List tables:** `yamchart tables --connection <name>` or filter with `--schema GOLD` / `--database ANALYTICS`
|
|
14
|
+
2. **Describe columns:** `yamchart describe DATABASE.SCHEMA.TABLE --connection <name>`
|
|
15
|
+
3. **Sample rows:** `yamchart sample DATABASE.SCHEMA.TABLE -l 10 --connection <name>` — Show sample rows (default: 5). Accepts dbt model names.
|
|
16
|
+
4. **Search:** `yamchart search <keyword> --connection <name>` — Find tables and columns matching keyword.
|
|
17
|
+
5. **Ad-hoc query:** `yamchart query "SELECT * FROM DATABASE.SCHEMA.TABLE LIMIT 10" --connection <name>`
|
|
18
|
+
6. **Write model** in `models/*.sql` using what you learned
|
|
19
|
+
|
|
20
|
+
If the user hasn't specified a connection, check `yamchart.yaml` for the default connection or list connections in `connections/*.yaml`.
|
|
21
|
+
|
|
22
|
+
## Snowflake-Specific Knowledge
|
|
23
|
+
|
|
24
|
+
**Naming:**
|
|
25
|
+
- Three-part naming: `DATABASE.SCHEMA.TABLE`
|
|
26
|
+
- Identifiers are UPPERCASE by default
|
|
27
|
+
- dbt model names often don't match materialized table paths (e.g., `marketing__fact_sessions` in dbt vs `ANALYTICS.GOLD.FACT_SESSIONS` in Snowflake)
|
|
28
|
+
|
|
29
|
+
**Column aliasing (critical):**
|
|
30
|
+
- Yamchart chart fields are case-sensitive and expect lowercase
|
|
31
|
+
- Always alias columns: `SELECT revenue AS "revenue", order_date AS "order_date"`
|
|
32
|
+
- Or use: `SELECT LOWER(col) ...` patterns where appropriate
|
|
33
|
+
|
|
34
|
+
**Semi-structured data:**
|
|
35
|
+
- `VARIANT`, `OBJECT`, `ARRAY` types are common
|
|
36
|
+
- Use `FLATTEN()` to unnest arrays: `SELECT f.value::STRING FROM table, LATERAL FLATTEN(input => array_col) f`
|
|
37
|
+
- Access nested fields: `col:key::STRING`, `col['key']::NUMBER`
|
|
38
|
+
- `TRY_CAST()` for safe type conversion
|
|
39
|
+
|
|
40
|
+
**Date/time:**
|
|
41
|
+
- `DATEADD(day, -7, CURRENT_DATE())`, `DATEDIFF(day, start, end)`
|
|
42
|
+
- `DATE_TRUNC('month', date_col)`
|
|
43
|
+
- Time travel: `SELECT * FROM table AT(OFFSET => -60*5)` (5 minutes ago)
|
|
44
|
+
|
|
45
|
+
**Performance tips:**
|
|
46
|
+
- Use `LIMIT` on large tables when exploring
|
|
47
|
+
- Prefer `WHERE` pushdown over subquery filtering
|
|
48
|
+
- `SAMPLE(100 ROWS)` for random sampling
|
|
49
|
+
- Check warehouse size if queries are slow
|
|
50
|
+
|
|
51
|
+
**Common patterns in yamchart models:**
|
|
52
|
+
```sql
|
|
53
|
+
-- @name: revenue_by_month
|
|
54
|
+
-- @param start_date: date = 2025-01-01
|
|
55
|
+
-- @param end_date: date = 2025-12-31
|
|
56
|
+
SELECT
|
|
57
|
+
DATE_TRUNC('month', order_date) AS "period",
|
|
58
|
+
SUM(amount) AS "revenue",
|
|
59
|
+
COUNT(DISTINCT customer_id) AS "customers"
|
|
60
|
+
FROM ANALYTICS.GOLD.ORDERS
|
|
61
|
+
WHERE order_date BETWEEN '{{ start_date }}' AND '{{ end_date }}'
|
|
62
|
+
GROUP BY 1
|
|
63
|
+
ORDER BY 1
|
|
64
|
+
```
|
|
@@ -29,6 +29,13 @@ yamchart advisor # AI-powered dbt model advisor
|
|
|
29
29
|
- `dashboards/*.yaml` — 12-column grid layouts with chart/text widgets, optional tabs
|
|
30
30
|
- `schedules/*.yaml` — cron-based Slack reports and threshold alerts
|
|
31
31
|
|
|
32
|
+
## Skills
|
|
33
|
+
|
|
34
|
+
This project includes Claude Code skills for database exploration and dbt model writing:
|
|
35
|
+
|
|
36
|
+
- `/dbt-advisor` — AI-powered dbt model advisor (no API key needed in Claude Code)
|
|
37
|
+
- `/duckdb`, `/postgres`, `/mysql`, `/snowflake`, `/databricks` — Database-specific exploration and model writing
|
|
38
|
+
|
|
32
39
|
## Yamchart Reference
|
|
33
40
|
|
|
34
41
|
See [docs/yamchart-reference.md](docs/yamchart-reference.md) for the full configuration reference including all chart types, parameters, dashboard tabs, filters, connections, schedules, authentication, and formatting options.
|