yamchart 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{advisor-CXXGXZ5V.js → advisor-IJG6UFTJ.js} +15 -7
- package/dist/advisor-IJG6UFTJ.js.map +1 -0
- package/dist/{chunk-ZN3AJM76.js → chunk-EHM6AMMA.js} +78 -2
- package/dist/chunk-EHM6AMMA.js.map +1 -0
- package/dist/chunk-WYS4ULBE.js +125 -0
- package/dist/chunk-WYS4ULBE.js.map +1 -0
- package/dist/{describe-Y4VD4JQX.js → describe-HD2RJQX3.js} +2 -2
- package/dist/{dev-7SWSX2X7.js → dev-LCYHM7UA.js} +19 -3
- package/dist/dev-LCYHM7UA.js.map +1 -0
- package/dist/{dist-ZRRM3OWF.js → dist-D5NLPC5X.js} +62 -1
- package/dist/dist-D5NLPC5X.js.map +1 -0
- package/dist/index.js +49 -6
- package/dist/index.js.map +1 -1
- package/dist/lineage-TX33DUDP.js +81 -0
- package/dist/lineage-TX33DUDP.js.map +1 -0
- package/dist/public/assets/{LoginPage-BhzP7Hkq.js → LoginPage-NNsbTBPo.js} +1 -1
- package/dist/public/assets/{PublicViewer-owhT_M21.js → PublicViewer-DDsvIFas.js} +1 -1
- package/dist/public/assets/{SetupWizard-BbV-9haQ.js → SetupWizard-CY-o51--.js} +1 -1
- package/dist/public/assets/{ShareManagement-BWDdV-hd.js → ShareManagement-B-Kz0tT8.js} +1 -1
- package/dist/public/assets/{UserManagement-DGGk6XAr.js → UserManagement-B6LjBcOb.js} +1 -1
- package/dist/public/assets/{index-Ce_q4or4.js → index-IDI-DDZi.js} +11 -11
- package/dist/public/assets/{index.es-DZ9bSPjd.js → index.es-DK8yr16j.js} +1 -1
- package/dist/public/assets/{jspdf.es.min-BZhCusuH.js → jspdf.es.min-CDLoTXR9.js} +3 -3
- package/dist/public/index.html +1 -1
- package/dist/{search-CXBNM2KK.js → search-ECGHSZBU.js} +2 -2
- package/dist/{sync-dbt-HICRXDB2.js → sync-dbt-RKEHTN4P.js} +35 -54
- package/dist/sync-dbt-RKEHTN4P.js.map +1 -0
- package/dist/sync-warehouse-WPNGI6YP.js +367 -0
- package/dist/sync-warehouse-WPNGI6YP.js.map +1 -0
- package/dist/{tables-ZY3NRC6E.js → tables-5BVEKUPB.js} +2 -2
- package/dist/templates/default/CLAUDE.md +2 -0
- package/dist/templates/default/docs/yamchart-reference.md +54 -0
- package/package.json +4 -4
- package/dist/advisor-CXXGXZ5V.js.map +0 -1
- package/dist/chunk-ZN3AJM76.js.map +0 -1
- package/dist/dev-7SWSX2X7.js.map +0 -1
- package/dist/dist-ZRRM3OWF.js.map +0 -1
- package/dist/sync-dbt-HICRXDB2.js.map +0 -1
- /package/dist/{describe-Y4VD4JQX.js.map → describe-HD2RJQX3.js.map} +0 -0
- /package/dist/{search-CXBNM2KK.js.map → search-ECGHSZBU.js.map} +0 -0
- /package/dist/{tables-ZY3NRC6E.js.map → tables-5BVEKUPB.js.map} +0 -0
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
generateCatalogJson,
|
|
3
|
+
generateCatalogMd
|
|
4
|
+
} from "./chunk-WYS4ULBE.js";
|
|
1
5
|
import "./chunk-DGUM43GV.js";
|
|
2
6
|
|
|
3
7
|
// src/commands/sync-dbt.ts
|
|
@@ -207,6 +211,36 @@ var LocalDbtSource = class {
|
|
|
207
211
|
model.table = node.database ? `${node.database}.${node.schema}.${tableName}` : `${node.schema}.${tableName}`;
|
|
208
212
|
}
|
|
209
213
|
}
|
|
214
|
+
for (const [nodeKey, node] of Object.entries(manifest.nodes)) {
|
|
215
|
+
const parts = nodeKey.split(".");
|
|
216
|
+
const modelName = parts[parts.length - 1];
|
|
217
|
+
const model = this.modelsCache.get(modelName);
|
|
218
|
+
if (!model || !node.depends_on?.nodes?.length) continue;
|
|
219
|
+
model.dependsOn = node.depends_on.nodes;
|
|
220
|
+
}
|
|
221
|
+
if (manifest.sources) {
|
|
222
|
+
for (const [sourceKey, source] of Object.entries(manifest.sources)) {
|
|
223
|
+
const sourceName = source.source_name ? `${source.source_name}.${source.name}` : source.name || sourceKey.split(".").slice(-2).join(".");
|
|
224
|
+
if (this.modelsCache.has(sourceName)) continue;
|
|
225
|
+
const tablePath = [source.database, source.schema, source.identifier || source.name].filter(Boolean).join(".");
|
|
226
|
+
const columns = source.columns ? Object.values(source.columns).map((col) => ({
|
|
227
|
+
name: col.name || "",
|
|
228
|
+
description: col.description || "",
|
|
229
|
+
data_type: col.type || "",
|
|
230
|
+
hints: []
|
|
231
|
+
})) : [];
|
|
232
|
+
this.modelsCache.set(sourceName, {
|
|
233
|
+
name: sourceName,
|
|
234
|
+
path: "",
|
|
235
|
+
description: source.description || "",
|
|
236
|
+
table: tablePath,
|
|
237
|
+
tags: [],
|
|
238
|
+
meta: {},
|
|
239
|
+
columns,
|
|
240
|
+
source: "dbt-source"
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
}
|
|
210
244
|
} catch {
|
|
211
245
|
}
|
|
212
246
|
}
|
|
@@ -373,59 +407,6 @@ function extractSourceFromSql(sql) {
|
|
|
373
407
|
return void 0;
|
|
374
408
|
}
|
|
375
409
|
|
|
376
|
-
// src/dbt/catalog.ts
|
|
377
|
-
function generateCatalogMd(data) {
|
|
378
|
-
const lines = [];
|
|
379
|
-
lines.push("# Data Catalog");
|
|
380
|
-
lines.push("");
|
|
381
|
-
lines.push(`> Source: ${data.source.type}:${data.source.path || data.source.repo || "unknown"}`);
|
|
382
|
-
lines.push(`> Last synced: ${data.syncedAt.split("T")[0]}`);
|
|
383
|
-
lines.push(`> Models: ${data.stats.modelsIncluded} included, ${data.stats.modelsExcluded} filtered out`);
|
|
384
|
-
lines.push("");
|
|
385
|
-
lines.push("---");
|
|
386
|
-
lines.push("");
|
|
387
|
-
lines.push("## Models");
|
|
388
|
-
lines.push("");
|
|
389
|
-
for (const model of data.models) {
|
|
390
|
-
lines.push(`### ${model.name}`);
|
|
391
|
-
lines.push("");
|
|
392
|
-
lines.push(model.description);
|
|
393
|
-
lines.push("");
|
|
394
|
-
if (model.table) {
|
|
395
|
-
lines.push(`**Table:** \`${model.table}\``);
|
|
396
|
-
}
|
|
397
|
-
if (model.tags.length > 0) {
|
|
398
|
-
lines.push(`**Tags:** ${model.tags.map((t) => `\`${t}\``).join(", ")}`);
|
|
399
|
-
}
|
|
400
|
-
lines.push("");
|
|
401
|
-
if (model.columns.length > 0) {
|
|
402
|
-
lines.push("| Column | Type | Description | Hints |");
|
|
403
|
-
lines.push("|--------|------|-------------|-------|");
|
|
404
|
-
for (const col of model.columns) {
|
|
405
|
-
const type = col.data_type || "";
|
|
406
|
-
const hints = col.hints.join(", ");
|
|
407
|
-
lines.push(`| ${col.name} | ${type} | ${col.description} | ${hints} |`);
|
|
408
|
-
}
|
|
409
|
-
lines.push("");
|
|
410
|
-
}
|
|
411
|
-
lines.push("**Yamchart models:**");
|
|
412
|
-
if (model.yamchartModels.length > 0) {
|
|
413
|
-
for (const ym of model.yamchartModels) {
|
|
414
|
-
lines.push(`- [\`${ym.name}\`](../${ym.path}) - ${ym.description || "No description"}`);
|
|
415
|
-
}
|
|
416
|
-
} else {
|
|
417
|
-
lines.push("None yet");
|
|
418
|
-
}
|
|
419
|
-
lines.push("");
|
|
420
|
-
lines.push("---");
|
|
421
|
-
lines.push("");
|
|
422
|
-
}
|
|
423
|
-
return lines.join("\n");
|
|
424
|
-
}
|
|
425
|
-
function generateCatalogJson(data) {
|
|
426
|
-
return JSON.stringify(data, null, 2);
|
|
427
|
-
}
|
|
428
|
-
|
|
429
410
|
// src/commands/sync-dbt.ts
|
|
430
411
|
async function loadSyncConfig(projectDir) {
|
|
431
412
|
const configPath = join4(projectDir, ".yamchart", "dbt-source.yaml");
|
|
@@ -565,4 +546,4 @@ export {
|
|
|
565
546
|
loadSyncConfig,
|
|
566
547
|
syncDbt
|
|
567
548
|
};
|
|
568
|
-
//# sourceMappingURL=sync-dbt-
|
|
549
|
+
//# sourceMappingURL=sync-dbt-RKEHTN4P.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/commands/sync-dbt.ts","../src/dbt/local-source.ts","../src/dbt/parser.ts","../src/dbt/scanner.ts"],"sourcesContent":["import { mkdir, writeFile, readFile, access } from 'fs/promises';\nimport { join } from 'path';\nimport { parse as parseYaml, stringify as stringifyYaml } from 'yaml';\nimport { LocalDbtSource } from '../dbt/local-source.js';\nimport { scanYamchartModels } from '../dbt/scanner.js';\nimport { generateCatalogMd, generateCatalogJson, type CatalogData, type CatalogModel } from '../dbt/catalog.js';\nimport type { DbtSourceConfig } from '../dbt/types.js';\n\nexport interface SyncDbtOptions {\n source: 'local' | 'github' | 'dbt-cloud';\n path?: string;\n repo?: string;\n branch?: string;\n include: string[];\n exclude: string[];\n tags: string[];\n refresh?: boolean;\n}\n\nexport interface SyncDbtResult {\n success: boolean;\n modelsIncluded: number;\n modelsExcluded: number;\n catalogPath: string;\n error?: string;\n}\n\n/**\n * Load saved sync config from .yamchart/dbt-source.yaml\n */\nexport async function loadSyncConfig(projectDir: string): Promise<DbtSourceConfig | null> {\n const configPath = join(projectDir, '.yamchart', 'dbt-source.yaml');\n\n try {\n await access(configPath);\n const content = await readFile(configPath, 'utf-8');\n return parseYaml(content) as DbtSourceConfig;\n } catch {\n return null;\n }\n}\n\n/**\n * Sync dbt project metadata to yamchart catalog.\n *\n * This function:\n * 1. Ensures .yamchart directory exists\n * 2. Handles refresh mode (load saved config)\n * 3. Validates source type (only 'local' supported for v1)\n * 4. Creates LocalDbtSource, lists models, applies filters\n * 5. Uses smart defaults if no filters specified\n * 6. Scans yamchart models for cross-references\n * 7. Generates catalog.md and catalog.json\n * 8. Saves sync config to dbt-source.yaml\n */\nexport async function syncDbt(\n projectDir: string,\n options: SyncDbtOptions\n): Promise<SyncDbtResult> {\n const yamchartDir = join(projectDir, '.yamchart');\n const catalogPath = join(yamchartDir, 'catalog.md');\n\n try {\n // Step 1: Ensure .yamchart directory exists\n await mkdir(yamchartDir, { recursive: true });\n\n // Step 2: Handle refresh mode - load saved config\n let effectiveOptions = { ...options };\n if (options.refresh) {\n const savedConfig = await loadSyncConfig(projectDir);\n if (!savedConfig) {\n return {\n success: false,\n modelsIncluded: 0,\n modelsExcluded: 0,\n catalogPath,\n error: 'No saved sync config found. Run sync-dbt with --path first.',\n };\n }\n\n effectiveOptions = {\n source: savedConfig.source,\n path: savedConfig.path,\n repo: savedConfig.repo,\n branch: savedConfig.branch,\n include: savedConfig.filters.include,\n exclude: savedConfig.filters.exclude,\n tags: savedConfig.filters.tags,\n };\n }\n\n // Step 3: Validate source type (only 'local' supported for v1)\n if (effectiveOptions.source !== 'local') {\n return {\n success: false,\n modelsIncluded: 0,\n modelsExcluded: 0,\n catalogPath,\n error: `Source type \"${effectiveOptions.source}\" not yet supported. Only \"local\" is available.`,\n };\n }\n\n if (!effectiveOptions.path) {\n return {\n success: false,\n modelsIncluded: 0,\n modelsExcluded: 0,\n catalogPath,\n error: 'Path to dbt project is required for local source.',\n };\n }\n\n // Step 4: Create LocalDbtSource, list models\n const dbtSource = new LocalDbtSource(effectiveOptions.path);\n const allModels = await dbtSource.listModels();\n\n // Step 5: Apply filters or use smart defaults\n let hasFilters =\n effectiveOptions.include.length > 0 ||\n effectiveOptions.exclude.length > 0 ||\n effectiveOptions.tags.length > 0;\n\n let filteredModels;\n if (hasFilters) {\n filteredModels = LocalDbtSource.filterModels(allModels, {\n include: effectiveOptions.include,\n exclude: effectiveOptions.exclude,\n tags: effectiveOptions.tags,\n });\n } else {\n // Use smart defaults - prefer marts/reporting, exclude staging/intermediate\n const defaults = LocalDbtSource.getDefaultFilters();\n const withDefaults = LocalDbtSource.filterModels(allModels, defaults);\n\n // If defaults filter out everything, include all models\n filteredModels = withDefaults.length > 0 ? withDefaults : allModels;\n }\n\n const modelsExcluded = allModels.length - filteredModels.length;\n\n // Get full model details for included models\n const modelNames = filteredModels.map(m => m.name);\n const fullModels = await dbtSource.getModels(modelNames);\n\n // Step 6: Scan yamchart models for cross-references\n const yamchartModels = await scanYamchartModels(projectDir);\n\n // Build catalog models with cross-references\n const catalogModels: CatalogModel[] = fullModels.map(model => {\n // Find yamchart models that reference this dbt model\n const referencingModels = yamchartModels.filter(ym =>\n ym.source === model.name || ym.source === model.table\n );\n\n return {\n ...model,\n yamchartModels: referencingModels,\n };\n });\n\n // Step 7: Generate catalog files\n const catalogData: CatalogData = {\n syncedAt: new Date().toISOString(),\n source: {\n type: effectiveOptions.source,\n path: effectiveOptions.path,\n repo: effectiveOptions.repo,\n },\n stats: {\n modelsIncluded: filteredModels.length,\n modelsExcluded,\n },\n models: catalogModels,\n };\n\n const catalogMd = generateCatalogMd(catalogData);\n const catalogJson = generateCatalogJson(catalogData);\n\n await writeFile(catalogPath, catalogMd, 'utf-8');\n await writeFile(join(yamchartDir, 'catalog.json'), catalogJson, 'utf-8');\n\n // Step 8: Save sync config for re-sync\n const syncConfig: DbtSourceConfig = {\n source: effectiveOptions.source,\n path: effectiveOptions.path,\n repo: effectiveOptions.repo,\n branch: effectiveOptions.branch,\n lastSync: catalogData.syncedAt,\n filters: {\n include: effectiveOptions.include,\n exclude: effectiveOptions.exclude,\n tags: effectiveOptions.tags,\n },\n stats: {\n modelsIncluded: filteredModels.length,\n modelsExcluded,\n },\n };\n\n const configYaml = stringifyYaml(syncConfig);\n await writeFile(join(yamchartDir, 'dbt-source.yaml'), configYaml, 'utf-8');\n\n return {\n success: true,\n modelsIncluded: filteredModels.length,\n modelsExcluded,\n catalogPath,\n };\n } catch (err) {\n return {\n success: false,\n modelsIncluded: 0,\n modelsExcluded: 0,\n catalogPath,\n error: err instanceof Error ? err.message : 'Unknown error during sync',\n };\n }\n}\n","import { readFile, access } from 'fs/promises';\nimport { join } from 'path';\nimport fg from 'fast-glob';\nimport { minimatch } from 'minimatch';\nimport { parseSchemaYml, parseProjectYml } from './parser.js';\nimport type {\n DbtSource,\n DbtProjectConfig,\n DbtModel,\n DbtModelSummary,\n} from './types.js';\n\nexport interface ModelFilters {\n include?: string[];\n exclude?: string[];\n tags?: string[];\n}\n\n/**\n * LocalDbtSource reads dbt project metadata from a local filesystem.\n * It parses schema.yml files to extract model definitions, columns, and hints.\n */\nexport class LocalDbtSource implements DbtSource {\n readonly type = 'local' as const;\n private projectPath: string;\n private modelsCache: Map<string, DbtModel> | null = null;\n private configCache: DbtProjectConfig | null = null;\n\n constructor(projectPath: string) {\n this.projectPath = projectPath;\n }\n\n /**\n * Read and parse dbt_project.yml\n */\n async getProjectConfig(): Promise<DbtProjectConfig> {\n if (this.configCache) {\n return this.configCache;\n }\n\n const configPath = join(this.projectPath, 'dbt_project.yml');\n const content = await readFile(configPath, 'utf-8');\n const parsed = parseProjectYml(content);\n\n this.configCache = {\n name: parsed.name,\n version: parsed.version,\n profile: parsed.profile,\n model_paths: parsed.modelPaths,\n target_path: 'target',\n vars: parsed.vars,\n };\n\n return this.configCache;\n }\n\n /**\n * Find all schema.yml files and parse models from them.\n * Returns summaries for model selection UI.\n */\n async listModels(): Promise<DbtModelSummary[]> {\n await this.loadModels();\n\n const summaries: DbtModelSummary[] = [];\n for (const model of this.modelsCache!.values()) {\n summaries.push({\n name: model.name,\n path: model.path,\n description: model.description || 'No description',\n tags: model.tags || [],\n });\n }\n\n return summaries;\n }\n\n /**\n * Get full model details by name.\n * @throws Error if model not found\n */\n async getModel(name: string): Promise<DbtModel> {\n await this.loadModels();\n\n const model = this.modelsCache!.get(name);\n if (!model) {\n throw new Error(`Model not found: ${name}`);\n }\n\n return model;\n }\n\n /**\n * Get multiple models by name.\n * @throws Error if any model not found\n */\n async getModels(names: string[]): Promise<DbtModel[]> {\n if (names.length === 0) {\n return [];\n }\n\n const models: DbtModel[] = [];\n for (const name of names) {\n models.push(await this.getModel(name));\n }\n\n return models;\n }\n\n /**\n * Load all models from schema.yml files into cache.\n */\n private async loadModels(): Promise<void> {\n if (this.modelsCache) {\n return;\n }\n\n const config = await this.getProjectConfig();\n this.modelsCache = new Map();\n\n // Find all schema.yml files in model paths\n for (const modelPath of config.model_paths) {\n const pattern = join(this.projectPath, modelPath, '**/*.yml');\n const files = await fg(pattern, {\n ignore: ['**/node_modules/**'],\n });\n\n for (const file of files) {\n const content = await readFile(file, 'utf-8');\n // Get relative path from project root\n const relativePath = file.slice(this.projectPath.length + 1);\n const models = parseSchemaYml(content, relativePath);\n\n for (const model of models) {\n this.modelsCache.set(model.name, model);\n }\n }\n }\n\n // Merge table paths from target/manifest.json (always present after dbt run)\n await this.mergeManifestPaths();\n // Merge column types from target/catalog.json if available (requires dbt docs generate)\n await this.mergeCatalogTypes();\n }\n\n /**\n * Read dbt's target/manifest.json and extract fully-qualified table paths.\n * manifest.json is always present after dbt run/build (unlike catalog.json\n * which requires dbt docs generate).\n */\n private async mergeManifestPaths(): Promise<void> {\n const manifestPath = join(this.projectPath, 'target', 'manifest.json');\n\n try {\n await access(manifestPath);\n } catch {\n return;\n }\n\n try {\n const content = await readFile(manifestPath, 'utf-8');\n const manifest = JSON.parse(content) as {\n nodes?: Record<string, {\n relation_name?: string;\n database?: string;\n schema?: string;\n alias?: string;\n name?: string;\n depends_on?: { nodes?: string[] };\n }>;\n sources?: Record<string, {\n database?: string;\n schema?: string;\n name?: string;\n identifier?: string;\n description?: string;\n columns?: Record<string, { name?: string; type?: string; description?: string }>;\n source_name?: string;\n }>;\n };\n\n if (!manifest.nodes) return;\n\n for (const [nodeKey, node] of Object.entries(manifest.nodes)) {\n const parts = nodeKey.split('.');\n const modelName = parts[parts.length - 1];\n\n const model = this.modelsCache!.get(modelName);\n if (!model || model.table) continue;\n\n // Prefer relation_name (e.g. '\"PROD\".\"ANALYTICS\".\"DIM_CUSTOMERS\"')\n if (node.relation_name) {\n // Strip quotes: \"PROD\".\"ANALYTICS\".\"DIM_CUSTOMERS\" → PROD.ANALYTICS.DIM_CUSTOMERS\n model.table = node.relation_name.replace(/\"/g, '');\n continue;\n }\n\n // Fall back to database/schema/alias fields\n const tableName = node.alias || node.name;\n if (node.schema && tableName) {\n model.table = node.database\n ? `${node.database}.${node.schema}.${tableName}`\n : `${node.schema}.${tableName}`;\n }\n }\n\n // Extract depends_on for each model\n for (const [nodeKey, node] of Object.entries(manifest.nodes)) {\n const parts = nodeKey.split('.');\n const modelName = parts[parts.length - 1];\n const model = this.modelsCache!.get(modelName);\n if (!model || !node.depends_on?.nodes?.length) continue;\n model.dependsOn = node.depends_on.nodes;\n }\n\n // Extract dbt sources from manifest.json sources section\n if (manifest.sources) {\n for (const [sourceKey, source] of Object.entries(manifest.sources)) {\n const sourceName = source.source_name\n ? `${source.source_name}.${source.name}`\n : source.name || sourceKey.split('.').slice(-2).join('.');\n\n // Skip if already in cache (model with same name takes precedence)\n if (this.modelsCache!.has(sourceName)) continue;\n\n const tablePath = [source.database, source.schema, source.identifier || source.name]\n .filter(Boolean)\n .join('.');\n\n const columns = source.columns\n ? Object.values(source.columns).map((col) => ({\n name: col.name || '',\n description: col.description || '',\n data_type: col.type || '',\n hints: [] as string[],\n }))\n : [];\n\n this.modelsCache!.set(sourceName, {\n name: sourceName,\n path: '',\n description: source.description || '',\n table: tablePath,\n tags: [],\n meta: {},\n columns,\n source: 'dbt-source',\n });\n }\n }\n } catch {\n // manifest.json unreadable or malformed — silently skip\n }\n }\n\n /**\n * Read dbt's target/catalog.json and merge column data_type, missing columns,\n * and table paths into loaded models.\n */\n private async mergeCatalogTypes(): Promise<void> {\n const catalogPath = join(this.projectPath, 'target', 'catalog.json');\n\n try {\n await access(catalogPath);\n } catch {\n return; // No catalog.json, nothing to merge\n }\n\n try {\n const content = await readFile(catalogPath, 'utf-8');\n const catalog = JSON.parse(content) as {\n nodes?: Record<string, {\n metadata?: { schema?: string; name?: string; database?: string };\n columns?: Record<string, { type?: string; name?: string }>;\n }>;\n };\n\n if (!catalog.nodes) return;\n\n // Build lookups: model name → column types, model name → table path\n const typesByModel = new Map<string, Map<string, string>>();\n const tableByModel = new Map<string, string>();\n\n for (const [nodeKey, node] of Object.entries(catalog.nodes)) {\n // Node keys are like \"model.project_name.model_name\"\n const parts = nodeKey.split('.');\n const modelName = parts[parts.length - 1];\n\n // Build fully-qualified table path from metadata\n if (node.metadata?.schema && node.metadata?.name) {\n const tablePath = node.metadata.database\n ? `${node.metadata.database}.${node.metadata.schema}.${node.metadata.name}`\n : `${node.metadata.schema}.${node.metadata.name}`;\n tableByModel.set(modelName, tablePath);\n }\n\n if (!node.columns) continue;\n\n const colTypes = new Map<string, string>();\n for (const [colName, colInfo] of Object.entries(node.columns)) {\n if (colInfo.type) {\n colTypes.set(colName.toLowerCase(), colInfo.type);\n }\n }\n\n if (colTypes.size > 0) {\n typesByModel.set(modelName, colTypes);\n }\n }\n\n // Merge into cached models\n for (const [modelName, model] of this.modelsCache!) {\n // Set table path from catalog if not already set in schema.yml\n const tablePath = tableByModel.get(modelName);\n if (tablePath && !model.table) {\n model.table = tablePath;\n }\n\n const colTypes = typesByModel.get(modelName);\n if (!colTypes) continue;\n\n // Merge types into existing columns\n const existingCols = new Set(model.columns.map(c => c.name.toLowerCase()));\n for (const column of model.columns) {\n if (!column.data_type) {\n const catalogType = colTypes.get(column.name.toLowerCase());\n if (catalogType) {\n column.data_type = catalogType;\n }\n }\n }\n\n // Add columns from catalog that don't exist in schema.yml\n for (const [colNameLower, colType] of colTypes) {\n if (!existingCols.has(colNameLower)) {\n model.columns.push({\n name: colNameLower,\n description: '',\n data_type: colType,\n hints: [],\n });\n }\n }\n }\n } catch {\n // catalog.json unreadable or malformed — silently skip\n }\n }\n\n /**\n * Filter models by include/exclude glob patterns and tags.\n * Patterns match against model paths.\n */\n static filterModels(\n models: DbtModelSummary[],\n filters: ModelFilters\n ): DbtModelSummary[] {\n let filtered = [...models];\n\n // Apply include patterns (match any)\n if (filters.include && filters.include.length > 0) {\n filtered = filtered.filter((model) =>\n filters.include!.some((pattern) => minimatch(model.path, pattern))\n );\n }\n\n // Apply exclude patterns (exclude any matches)\n if (filters.exclude && filters.exclude.length > 0) {\n filtered = filtered.filter(\n (model) =>\n !filters.exclude!.some((pattern) => minimatch(model.path, pattern))\n );\n }\n\n // Apply tag filters (match any)\n if (filters.tags && filters.tags.length > 0) {\n filtered = filtered.filter((model) =>\n filters.tags!.some((tag) => model.tags.includes(tag))\n );\n }\n\n return filtered;\n }\n\n /**\n * Get default filters that focus on marts/reporting models.\n * These are typically the models most useful for BI dashboards.\n */\n static getDefaultFilters(): Required<ModelFilters> {\n return {\n include: ['**/marts/**', '**/reporting/**'],\n exclude: ['**/staging/**', '**/intermediate/**'],\n tags: [],\n };\n }\n}\n","import { parse as parseYaml } from 'yaml';\nimport { dirname, join } from 'path';\nimport type { DbtModel, DbtColumn } from './types.js';\n\ninterface RawSchemaYml {\n version: number;\n models?: RawModel[];\n}\n\ninterface RawModel {\n name: string;\n description?: string;\n meta?: Record<string, unknown>;\n tags?: string[];\n columns?: RawColumn[];\n}\n\ninterface RawColumn {\n name: string;\n description?: string;\n data_type?: string;\n tests?: (string | Record<string, unknown>)[];\n}\n\n/**\n * Extract hints from dbt column tests.\n * - unique → \"unique\"\n * - not_null → \"required\"\n * - relationships: { to: ref('X') } → \"fk:X\"\n */\nexport function extractHintsFromTests(tests: (string | Record<string, unknown>)[]): string[] {\n const hints: string[] = [];\n\n for (const test of tests) {\n if (typeof test === 'string') {\n if (test === 'unique') {\n hints.push('unique');\n } else if (test === 'not_null') {\n hints.push('required');\n } else if (test === 'primary_key') {\n hints.push('primary_key');\n }\n } else if (typeof test === 'object' && test !== null) {\n // Handle relationships test\n if ('relationships' in test) {\n const rel = test.relationships as { to?: string; field?: string };\n if (rel.to) {\n // Extract table name from ref('table_name')\n const match = rel.to.match(/ref\\(['\"]([^'\"]+)['\"]\\)/);\n if (match) {\n hints.push(`fk:${match[1]}`);\n }\n }\n }\n // Handle dbt_constraints for primary key\n if ('dbt_constraints' in test) {\n const constraint = test.dbt_constraints as { type?: string };\n if (constraint.type === 'primary_key') {\n hints.push('primary_key');\n }\n }\n }\n }\n\n return hints;\n}\n\n/**\n * Parse a dbt schema.yml file and extract model definitions.\n * @param content - Raw YAML content\n * @param schemaPath - Path to the schema file (e.g., \"models/marts/_schema.yml\")\n * @returns Array of parsed models\n */\nexport function parseSchemaYml(content: string, schemaPath: string): DbtModel[] {\n const parsed = parseYaml(content) as RawSchemaYml;\n\n if (!parsed?.models || !Array.isArray(parsed.models)) {\n return [];\n }\n\n const schemaDir = dirname(schemaPath);\n\n return parsed.models.map((model): DbtModel => {\n const columns: DbtColumn[] = (model.columns || []).map((col) => ({\n name: col.name,\n description: col.description || '',\n data_type: col.data_type,\n hints: col.tests ? extractHintsFromTests(col.tests) : [],\n }));\n\n return {\n name: model.name,\n path: join(schemaDir, `${model.name}.sql`),\n description: model.description || 'No description',\n tags: model.tags || [],\n meta: model.meta || {},\n columns,\n };\n });\n}\n\n/**\n * Parse dbt_project.yml to get project-level config.\n */\nexport function parseProjectYml(content: string): {\n name: string;\n version?: string;\n profile?: string;\n modelPaths: string[];\n vars: Record<string, unknown>;\n} {\n const parsed = parseYaml(content) as Record<string, unknown>;\n\n return {\n name: (parsed.name as string) || 'unknown',\n version: parsed.version as string | undefined,\n profile: parsed.profile as string | undefined,\n modelPaths: (parsed['model-paths'] as string[]) || (parsed.model_paths as string[]) || ['models'],\n vars: (parsed.vars as Record<string, unknown>) || {},\n };\n}\n","import { readFile, access, readdir } from 'fs/promises';\nimport { join, extname, relative } from 'path';\n\nexport interface YamchartModel {\n name: string;\n description: string;\n path: string; // relative to project\n source?: string; // dbt table this queries\n}\n\n/**\n * Scan yamchart models directory and extract metadata.\n * Used to cross-reference which yamchart models use which dbt tables.\n */\nexport async function scanYamchartModels(projectDir: string): Promise<YamchartModel[]> {\n const modelsDir = join(projectDir, 'models');\n const models: YamchartModel[] = [];\n\n try {\n await access(modelsDir);\n } catch {\n return [];\n }\n\n await scanModelsRecursive(modelsDir, projectDir, models);\n return models;\n}\n\nasync function scanModelsRecursive(\n dir: string,\n projectDir: string,\n models: YamchartModel[]\n): Promise<void> {\n const entries = await readdir(dir, { withFileTypes: true });\n\n for (const entry of entries) {\n const fullPath = join(dir, entry.name);\n\n if (entry.isDirectory()) {\n await scanModelsRecursive(fullPath, projectDir, models);\n } else if (extname(entry.name) === '.sql') {\n const content = await readFile(fullPath, 'utf-8');\n const metadata = parseModelMetadata(content);\n\n if (metadata.name) {\n models.push({\n name: metadata.name,\n description: metadata.description || '',\n path: relative(projectDir, fullPath),\n source: metadata.source || extractSourceFromSql(content),\n });\n }\n }\n }\n}\n\ninterface ModelMetadata {\n name?: string;\n description?: string;\n source?: string;\n}\n\n/**\n * Parse yamchart model metadata from SQL comments.\n */\nfunction parseModelMetadata(content: string): ModelMetadata {\n const metadata: ModelMetadata = {};\n\n // Match @name: value\n const nameMatch = content.match(/--\\s*@name:\\s*(.+)/);\n if (nameMatch) {\n metadata.name = nameMatch[1].trim();\n }\n\n // Match @description: value\n const descMatch = content.match(/--\\s*@description:\\s*(.+)/);\n if (descMatch) {\n metadata.description = descMatch[1].trim();\n }\n\n // Match @source: value (explicit dbt table reference)\n const sourceMatch = content.match(/--\\s*@source:\\s*(.+)/);\n if (sourceMatch) {\n metadata.source = sourceMatch[1].trim();\n }\n\n return metadata;\n}\n\n/**\n * Extract the primary table name from SQL FROM clause.\n * This is a best-effort extraction for cross-referencing.\n */\nfunction extractSourceFromSql(sql: string): string | undefined {\n // Remove comments\n const noComments = sql.replace(/--.*$/gm, '').replace(/\\/\\*[\\s\\S]*?\\*\\//g, '');\n\n // Match FROM table_name (handles schema.table and database.schema.table)\n const fromMatch = noComments.match(/\\bFROM\\s+(\\{\\{\\s*ref\\(['\"]([^'\"]+)['\"]\\)\\s*\\}\\}|[\\w.]+)/i);\n\n if (fromMatch) {\n // If it's a Jinja ref(), extract the table name\n if (fromMatch[2]) {\n return fromMatch[2];\n }\n // Otherwise return the raw table name\n return fromMatch[1];\n }\n\n return undefined;\n}\n"],"mappings":";;;;;;;AAAA,SAAS,OAAO,WAAW,YAAAA,WAAU,UAAAC,eAAc;AACnD,SAAS,QAAAC,aAAY;AACrB,SAAS,SAASC,YAAW,aAAa,qBAAqB;;;ACF/D,SAAS,UAAU,cAAc;AACjC,SAAS,QAAAC,aAAY;AACrB,OAAO,QAAQ;AACf,SAAS,iBAAiB;;;ACH1B,SAAS,SAAS,iBAAiB;AACnC,SAAS,SAAS,YAAY;AA6BvB,SAAS,sBAAsB,OAAuD;AAC3F,QAAM,QAAkB,CAAC;AAEzB,aAAW,QAAQ,OAAO;AACxB,QAAI,OAAO,SAAS,UAAU;AAC5B,UAAI,SAAS,UAAU;AACrB,cAAM,KAAK,QAAQ;AAAA,MACrB,WAAW,SAAS,YAAY;AAC9B,cAAM,KAAK,UAAU;AAAA,MACvB,WAAW,SAAS,eAAe;AACjC,cAAM,KAAK,aAAa;AAAA,MAC1B;AAAA,IACF,WAAW,OAAO,SAAS,YAAY,SAAS,MAAM;AAEpD,UAAI,mBAAmB,MAAM;AAC3B,cAAM,MAAM,KAAK;AACjB,YAAI,IAAI,IAAI;AAEV,gBAAM,QAAQ,IAAI,GAAG,MAAM,yBAAyB;AACpD,cAAI,OAAO;AACT,kBAAM,KAAK,MAAM,MAAM,CAAC,CAAC,EAAE;AAAA,UAC7B;AAAA,QACF;AAAA,MACF;AAEA,UAAI,qBAAqB,MAAM;AAC7B,cAAM,aAAa,KAAK;AACxB,YAAI,WAAW,SAAS,eAAe;AACrC,gBAAM,KAAK,aAAa;AAAA,QAC1B;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAQO,SAAS,eAAe,SAAiB,YAAgC;AAC9E,QAAM,SAAS,UAAU,OAAO;AAEhC,MAAI,CAAC,QAAQ,UAAU,CAAC,MAAM,QAAQ,OAAO,MAAM,GAAG;AACpD,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,YAAY,QAAQ,UAAU;AAEpC,SAAO,OAAO,OAAO,IAAI,CAAC,UAAoB;AAC5C,UAAM,WAAwB,MAAM,WAAW,CAAC,GAAG,IAAI,CAAC,SAAS;AAAA,MAC/D,MAAM,IAAI;AAAA,MACV,aAAa,IAAI,eAAe;AAAA,MAChC,WAAW,IAAI;AAAA,MACf,OAAO,IAAI,QAAQ,sBAAsB,IAAI,KAAK,IAAI,CAAC;AAAA,IACzD,EAAE;AAEF,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,MAAM,KAAK,WAAW,GAAG,MAAM,IAAI,MAAM;AAAA,MACzC,aAAa,MAAM,eAAe;AAAA,MAClC,MAAM,MAAM,QAAQ,CAAC;AAAA,MACrB,MAAM,MAAM,QAAQ,CAAC;AAAA,MACrB;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAKO,SAAS,gBAAgB,SAM9B;AACA,QAAM,SAAS,UAAU,OAAO;AAEhC,SAAO;AAAA,IACL,MAAO,OAAO,QAAmB;AAAA,IACjC,SAAS,OAAO;AAAA,IAChB,SAAS,OAAO;AAAA,IAChB,YAAa,OAAO,aAAa,KAAmB,OAAO,eAA4B,CAAC,QAAQ;AAAA,IAChG,MAAO,OAAO,QAAoC,CAAC;AAAA,EACrD;AACF;;;ADlGO,IAAM,iBAAN,MAA0C;AAAA,EACtC,OAAO;AAAA,EACR;AAAA,EACA,cAA4C;AAAA,EAC5C,cAAuC;AAAA,EAE/C,YAAY,aAAqB;AAC/B,SAAK,cAAc;AAAA,EACrB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,mBAA8C;AAClD,QAAI,KAAK,aAAa;AACpB,aAAO,KAAK;AAAA,IACd;AAEA,UAAM,aAAaC,MAAK,KAAK,aAAa,iBAAiB;AAC3D,UAAM,UAAU,MAAM,SAAS,YAAY,OAAO;AAClD,UAAM,SAAS,gBAAgB,OAAO;AAEtC,SAAK,cAAc;AAAA,MACjB,MAAM,OAAO;AAAA,MACb,SAAS,OAAO;AAAA,MAChB,SAAS,OAAO;AAAA,MAChB,aAAa,OAAO;AAAA,MACpB,aAAa;AAAA,MACb,MAAM,OAAO;AAAA,IACf;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,aAAyC;AAC7C,UAAM,KAAK,WAAW;AAEtB,UAAM,YAA+B,CAAC;AACtC,eAAW,SAAS,KAAK,YAAa,OAAO,GAAG;AAC9C,gBAAU,KAAK;AAAA,QACb,MAAM,MAAM;AAAA,QACZ,MAAM,MAAM;AAAA,QACZ,aAAa,MAAM,eAAe;AAAA,QAClC,MAAM,MAAM,QAAQ,CAAC;AAAA,MACvB,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,SAAS,MAAiC;AAC9C,UAAM,KAAK,WAAW;AAEtB,UAAM,QAAQ,KAAK,YAAa,IAAI,IAAI;AACxC,QAAI,CAAC,OAAO;AACV,YAAM,IAAI,MAAM,oBAAoB,IAAI,EAAE;AAAA,IAC5C;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,UAAU,OAAsC;AACpD,QAAI,MAAM,WAAW,GAAG;AACtB,aAAO,CAAC;AAAA,IACV;AAEA,UAAM,SAAqB,CAAC;AAC5B,eAAW,QAAQ,OAAO;AACxB,aAAO,KAAK,MAAM,KAAK,SAAS,IAAI,CAAC;AAAA,IACvC;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,aAA4B;AACxC,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AAEA,UAAM,SAAS,MAAM,KAAK,iBAAiB;AAC3C,SAAK,cAAc,oBAAI,IAAI;AAG3B,eAAW,aAAa,OAAO,aAAa;AAC1C,YAAM,UAAUA,MAAK,KAAK,aAAa,WAAW,UAAU;AAC5D,YAAM,QAAQ,MAAM,GAAG,SAAS;AAAA,QAC9B,QAAQ,CAAC,oBAAoB;AAAA,MAC/B,CAAC;AAED,iBAAW,QAAQ,OAAO;AACxB,cAAM,UAAU,MAAM,SAAS,MAAM,OAAO;AAE5C,cAAM,eAAe,KAAK,MAAM,KAAK,YAAY,SAAS,CAAC;AAC3D,cAAM,SAAS,eAAe,SAAS,YAAY;AAEnD,mBAAW,SAAS,QAAQ;AAC1B,eAAK,YAAY,IAAI,MAAM,MAAM,KAAK;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAGA,UAAM,KAAK,mBAAmB;AAE9B,UAAM,KAAK,kBAAkB;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,qBAAoC;AAChD,UAAM,eAAeA,MAAK,KAAK,aAAa,UAAU,eAAe;AAErE,QAAI;AACF,YAAM,OAAO,YAAY;AAAA,IAC3B,QAAQ;AACN;AAAA,IACF;AAEA,QAAI;AACF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,WAAW,KAAK,MAAM,OAAO;AAoBnC,UAAI,CAAC,SAAS,MAAO;AAErB,iBAAW,CAAC,SAAS,IAAI,KAAK,OAAO,QAAQ,SAAS,KAAK,GAAG;AAC5D,cAAM,QAAQ,QAAQ,MAAM,GAAG;AAC/B,cAAM,YAAY,MAAM,MAAM,SAAS,CAAC;AAExC,cAAM,QAAQ,KAAK,YAAa,IAAI,SAAS;AAC7C,YAAI,CAAC,SAAS,MAAM,MAAO;AAG3B,YAAI,KAAK,eAAe;AAEtB,gBAAM,QAAQ,KAAK,cAAc,QAAQ,MAAM,EAAE;AACjD;AAAA,QACF;AAGA,cAAM,YAAY,KAAK,SAAS,KAAK;AACrC,YAAI,KAAK,UAAU,WAAW;AAC5B,gBAAM,QAAQ,KAAK,WACf,GAAG,KAAK,QAAQ,IAAI,KAAK,MAAM,IAAI,SAAS,KAC5C,GAAG,KAAK,MAAM,IAAI,SAAS;AAAA,QACjC;AAAA,MACF;AAGA,iBAAW,CAAC,SAAS,IAAI,KAAK,OAAO,QAAQ,SAAS,KAAK,GAAG;AAC5D,cAAM,QAAQ,QAAQ,MAAM,GAAG;AAC/B,cAAM,YAAY,MAAM,MAAM,SAAS,CAAC;AACxC,cAAM,QAAQ,KAAK,YAAa,IAAI,SAAS;AAC7C,YAAI,CAAC,SAAS,CAAC,KAAK,YAAY,OAAO,OAAQ;AAC/C,cAAM,YAAY,KAAK,WAAW;AAAA,MACpC;AAGA,UAAI,SAAS,SAAS;AACpB,mBAAW,CAAC,WAAW,MAAM,KAAK,OAAO,QAAQ,SAAS,OAAO,GAAG;AAClE,gBAAM,aAAa,OAAO,cACtB,GAAG,OAAO,WAAW,IAAI,OAAO,IAAI,KACpC,OAAO,QAAQ,UAAU,MAAM,GAAG,EAAE,MAAM,EAAE,EAAE,KAAK,GAAG;AAG1D,cAAI,KAAK,YAAa,IAAI,UAAU,EAAG;AAEvC,gBAAM,YAAY,CAAC,OAAO,UAAU,OAAO,QAAQ,OAAO,cAAc,OAAO,IAAI,EAChF,OAAO,OAAO,EACd,KAAK,GAAG;AAEX,gBAAM,UAAU,OAAO,UACnB,OAAO,OAAO,OAAO,OAAO,EAAE,IAAI,CAAC,SAAS;AAAA,YAC1C,MAAM,IAAI,QAAQ;AAAA,YAClB,aAAa,IAAI,eAAe;AAAA,YAChC,WAAW,IAAI,QAAQ;AAAA,YACvB,OAAO,CAAC;AAAA,UACV,EAAE,IACF,CAAC;AAEL,eAAK,YAAa,IAAI,YAAY;AAAA,YAChC,MAAM;AAAA,YACN,MAAM;AAAA,YACN,aAAa,OAAO,eAAe;AAAA,YACnC,OAAO;AAAA,YACP,MAAM,CAAC;AAAA,YACP,MAAM,CAAC;AAAA,YACP;AAAA,YACA,QAAQ;AAAA,UACV,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,oBAAmC;AAC/C,UAAM,cAAcA,MAAK,KAAK,aAAa,UAAU,cAAc;AAEnE,QAAI;AACF,YAAM,OAAO,WAAW;AAAA,IAC1B,QAAQ;AACN;AAAA,IACF;AAEA,QAAI;AACF,YAAM,UAAU,MAAM,SAAS,aAAa,OAAO;AACnD,YAAM,UAAU,KAAK,MAAM,OAAO;AAOlC,UAAI,CAAC,QAAQ,MAAO;AAGpB,YAAM,eAAe,oBAAI,IAAiC;AAC1D,YAAM,eAAe,oBAAI,IAAoB;AAE7C,iBAAW,CAAC,SAAS,IAAI,KAAK,OAAO,QAAQ,QAAQ,KAAK,GAAG;AAE3D,cAAM,QAAQ,QAAQ,MAAM,GAAG;AAC/B,cAAM,YAAY,MAAM,MAAM,SAAS,CAAC;AAGxC,YAAI,KAAK,UAAU,UAAU,KAAK,UAAU,MAAM;AAChD,gBAAM,YAAY,KAAK,SAAS,WAC5B,GAAG,KAAK,SAAS,QAAQ,IAAI,KAAK,SAAS,MAAM,IAAI,KAAK,SAAS,IAAI,KACvE,GAAG,KAAK,SAAS,MAAM,IAAI,KAAK,SAAS,IAAI;AACjD,uBAAa,IAAI,WAAW,SAAS;AAAA,QACvC;AAEA,YAAI,CAAC,KAAK,QAAS;AAEnB,cAAM,WAAW,oBAAI,IAAoB;AACzC,mBAAW,CAAC,SAAS,OAAO,KAAK,OAAO,QAAQ,KAAK,OAAO,GAAG;AAC7D,cAAI,QAAQ,MAAM;AAChB,qBAAS,IAAI,QAAQ,YAAY,GAAG,QAAQ,IAAI;AAAA,UAClD;AAAA,QACF;AAEA,YAAI,SAAS,OAAO,GAAG;AACrB,uBAAa,IAAI,WAAW,QAAQ;AAAA,QACtC;AAAA,MACF;AAGA,iBAAW,CAAC,WAAW,KAAK,KAAK,KAAK,aAAc;AAElD,cAAM,YAAY,aAAa,IAAI,SAAS;AAC5C,YAAI,aAAa,CAAC,MAAM,OAAO;AAC7B,gBAAM,QAAQ;AAAA,QAChB;AAEA,cAAM,WAAW,aAAa,IAAI,SAAS;AAC3C,YAAI,CAAC,SAAU;AAGf,cAAM,eAAe,IAAI,IAAI,MAAM,QAAQ,IAAI,OAAK,EAAE,KAAK,YAAY,CAAC,CAAC;AACzE,mBAAW,UAAU,MAAM,SAAS;AAClC,cAAI,CAAC,OAAO,WAAW;AACrB,kBAAM,cAAc,SAAS,IAAI,OAAO,KAAK,YAAY,CAAC;AAC1D,gBAAI,aAAa;AACf,qBAAO,YAAY;AAAA,YACrB;AAAA,UACF;AAAA,QACF;AAGA,mBAAW,CAAC,cAAc,OAAO,KAAK,UAAU;AAC9C,cAAI,CAAC,aAAa,IAAI,YAAY,GAAG;AACnC,kBAAM,QAAQ,KAAK;AAAA,cACjB,MAAM;AAAA,cACN,aAAa;AAAA,cACb,WAAW;AAAA,cACX,OAAO,CAAC;AAAA,YACV,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,OAAO,aACL,QACA,SACmB;AACnB,QAAI,WAAW,CAAC,GAAG,MAAM;AAGzB,QAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAAG;AACjD,iBAAW,SAAS;AAAA,QAAO,CAAC,UAC1B,QAAQ,QAAS,KAAK,CAAC,YAAY,UAAU,MAAM,MAAM,OAAO,CAAC;AAAA,MACnE;AAAA,IACF;AAGA,QAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAAG;AACjD,iBAAW,SAAS;AAAA,QAClB,CAAC,UACC,CAAC,QAAQ,QAAS,KAAK,CAAC,YAAY,UAAU,MAAM,MAAM,OAAO,CAAC;AAAA,MACtE;AAAA,IACF;AAGA,QAAI,QAAQ,QAAQ,QAAQ,KAAK,SAAS,GAAG;AAC3C,iBAAW,SAAS;AAAA,QAAO,CAAC,UAC1B,QAAQ,KAAM,KAAK,CAAC,QAAQ,MAAM,KAAK,SAAS,GAAG,CAAC;AAAA,MACtD;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,OAAO,oBAA4C;AACjD,WAAO;AAAA,MACL,SAAS,CAAC,eAAe,iBAAiB;AAAA,MAC1C,SAAS,CAAC,iBAAiB,oBAAoB;AAAA,MAC/C,MAAM,CAAC;AAAA,IACT;AAAA,EACF;AACF;;;AE1YA,SAAS,YAAAC,WAAU,UAAAC,SAAQ,eAAe;AAC1C,SAAS,QAAAC,OAAM,SAAS,gBAAgB;AAaxC,eAAsB,mBAAmB,YAA8C;AACrF,QAAM,YAAYA,MAAK,YAAY,QAAQ;AAC3C,QAAM,SAA0B,CAAC;AAEjC,MAAI;AACF,UAAMD,QAAO,SAAS;AAAA,EACxB,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,WAAW,YAAY,MAAM;AACvD,SAAO;AACT;AAEA,eAAe,oBACb,KACA,YACA,QACe;AACf,QAAM,UAAU,MAAM,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;AAE1D,aAAW,SAAS,SAAS;AAC3B,UAAM,WAAWC,MAAK,KAAK,MAAM,IAAI;AAErC,QAAI,MAAM,YAAY,GAAG;AACvB,YAAM,oBAAoB,UAAU,YAAY,MAAM;AAAA,IACxD,WAAW,QAAQ,MAAM,IAAI,MAAM,QAAQ;AACzC,YAAM,UAAU,MAAMF,UAAS,UAAU,OAAO;AAChD,YAAM,WAAW,mBAAmB,OAAO;AAE3C,UAAI,SAAS,MAAM;AACjB,eAAO,KAAK;AAAA,UACV,MAAM,SAAS;AAAA,UACf,aAAa,SAAS,eAAe;AAAA,UACrC,MAAM,SAAS,YAAY,QAAQ;AAAA,UACnC,QAAQ,SAAS,UAAU,qBAAqB,OAAO;AAAA,QACzD,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;AAWA,SAAS,mBAAmB,SAAgC;AAC1D,QAAM,WAA0B,CAAC;AAGjC,QAAM,YAAY,QAAQ,MAAM,oBAAoB;AACpD,MAAI,WAAW;AACb,aAAS,OAAO,UAAU,CAAC,EAAE,KAAK;AAAA,EACpC;AAGA,QAAM,YAAY,QAAQ,MAAM,2BAA2B;AAC3D,MAAI,WAAW;AACb,aAAS,cAAc,UAAU,CAAC,EAAE,KAAK;AAAA,EAC3C;AAGA,QAAM,cAAc,QAAQ,MAAM,sBAAsB;AACxD,MAAI,aAAa;AACf,aAAS,SAAS,YAAY,CAAC,EAAE,KAAK;AAAA,EACxC;AAEA,SAAO;AACT;AAMA,SAAS,qBAAqB,KAAiC;AAE7D,QAAM,aAAa,IAAI,QAAQ,WAAW,EAAE,EAAE,QAAQ,qBAAqB,EAAE;AAG7E,QAAM,YAAY,WAAW,MAAM,0DAA0D;AAE7F,MAAI,WAAW;AAEb,QAAI,UAAU,CAAC,GAAG;AAChB,aAAO,UAAU,CAAC;AAAA,IACpB;AAEA,WAAO,UAAU,CAAC;AAAA,EACpB;AAEA,SAAO;AACT;;;AHhFA,eAAsB,eAAe,YAAqD;AACxF,QAAM,aAAaG,MAAK,YAAY,aAAa,iBAAiB;AAElE,MAAI;AACF,UAAMC,QAAO,UAAU;AACvB,UAAM,UAAU,MAAMC,UAAS,YAAY,OAAO;AAClD,WAAOC,WAAU,OAAO;AAAA,EAC1B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAeA,eAAsB,QACpB,YACA,SACwB;AACxB,QAAM,cAAcH,MAAK,YAAY,WAAW;AAChD,QAAM,cAAcA,MAAK,aAAa,YAAY;AAElD,MAAI;AAEF,UAAM,MAAM,aAAa,EAAE,WAAW,KAAK,CAAC;AAG5C,QAAI,mBAAmB,EAAE,GAAG,QAAQ;AACpC,QAAI,QAAQ,SAAS;AACnB,YAAM,cAAc,MAAM,eAAe,UAAU;AACnD,UAAI,CAAC,aAAa;AAChB,eAAO;AAAA,UACL,SAAS;AAAA,UACT,gBAAgB;AAAA,UAChB,gBAAgB;AAAA,UAChB;AAAA,UACA,OAAO;AAAA,QACT;AAAA,MACF;AAEA,yBAAmB;AAAA,QACjB,QAAQ,YAAY;AAAA,QACpB,MAAM,YAAY;AAAA,QAClB,MAAM,YAAY;AAAA,QAClB,QAAQ,YAAY;AAAA,QACpB,SAAS,YAAY,QAAQ;AAAA,QAC7B,SAAS,YAAY,QAAQ;AAAA,QAC7B,MAAM,YAAY,QAAQ;AAAA,MAC5B;AAAA,IACF;AAGA,QAAI,iBAAiB,WAAW,SAAS;AACvC,aAAO;AAAA,QACL,SAAS;AAAA,QACT,gBAAgB;AAAA,QAChB,gBAAgB;AAAA,QAChB;AAAA,QACA,OAAO,gBAAgB,iBAAiB,MAAM;AAAA,MAChD;AAAA,IACF;AAEA,QAAI,CAAC,iBAAiB,MAAM;AAC1B,aAAO;AAAA,QACL,SAAS;AAAA,QACT,gBAAgB;AAAA,QAChB,gBAAgB;AAAA,QAChB;AAAA,QACA,OAAO;AAAA,MACT;AAAA,IACF;AAGA,UAAM,YAAY,IAAI,eAAe,iBAAiB,IAAI;AAC1D,UAAM,YAAY,MAAM,UAAU,WAAW;AAG7C,QAAI,aACF,iBAAiB,QAAQ,SAAS,KAClC,iBAAiB,QAAQ,SAAS,KAClC,iBAAiB,KAAK,SAAS;AAEjC,QAAI;AACJ,QAAI,YAAY;AACd,uBAAiB,eAAe,aAAa,WAAW;AAAA,QACtD,SAAS,iBAAiB;AAAA,QAC1B,SAAS,iBAAiB;AAAA,QAC1B,MAAM,iBAAiB;AAAA,MACzB,CAAC;AAAA,IACH,OAAO;AAEL,YAAM,WAAW,eAAe,kBAAkB;AAClD,YAAM,eAAe,eAAe,aAAa,WAAW,QAAQ;AAGpE,uBAAiB,aAAa,SAAS,IAAI,eAAe;AAAA,IAC5D;AAEA,UAAM,iBAAiB,UAAU,SAAS,eAAe;AAGzD,UAAM,aAAa,eAAe,IAAI,OAAK,EAAE,IAAI;AACjD,UAAM,aAAa,MAAM,UAAU,UAAU,UAAU;AAGvD,UAAM,iBAAiB,MAAM,mBAAmB,UAAU;AAG1D,UAAM,gBAAgC,WAAW,IAAI,WAAS;AAE5D,YAAM,oBAAoB,eAAe;AAAA,QAAO,QAC9C,GAAG,WAAW,MAAM,QAAQ,GAAG,WAAW,MAAM;AAAA,MAClD;AAEA,aAAO;AAAA,QACL,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,IACF,CAAC;AAGD,UAAM,cAA2B;AAAA,MAC/B,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,MACjC,QAAQ;AAAA,QACN,MAAM,iBAAiB;AAAA,QACvB,MAAM,iBAAiB;AAAA,QACvB,MAAM,iBAAiB;AAAA,MACzB;AAAA,MACA,OAAO;AAAA,QACL,gBAAgB,eAAe;AAAA,QAC/B;AAAA,MACF;AAAA,MACA,QAAQ;AAAA,IACV;AAEA,UAAM,YAAY,kBAAkB,WAAW;AAC/C,UAAM,cAAc,oBAAoB,WAAW;AAEnD,UAAM,UAAU,aAAa,WAAW,OAAO;AAC/C,UAAM,UAAUA,MAAK,aAAa,cAAc,GAAG,aAAa,OAAO;AAGvE,UAAM,aAA8B;AAAA,MAClC,QAAQ,iBAAiB;AAAA,MACzB,MAAM,iBAAiB;AAAA,MACvB,MAAM,iBAAiB;AAAA,MACvB,QAAQ,iBAAiB;AAAA,MACzB,UAAU,YAAY;AAAA,MACtB,SAAS;AAAA,QACP,SAAS,iBAAiB;AAAA,QAC1B,SAAS,iBAAiB;AAAA,QAC1B,MAAM,iBAAiB;AAAA,MACzB;AAAA,MACA,OAAO;AAAA,QACL,gBAAgB,eAAe;AAAA,QAC/B;AAAA,MACF;AAAA,IACF;AAEA,UAAM,aAAa,cAAc,UAAU;AAC3C,UAAM,UAAUA,MAAK,aAAa,iBAAiB,GAAG,YAAY,OAAO;AAEzE,WAAO;AAAA,MACL,SAAS;AAAA,MACT,gBAAgB,eAAe;AAAA,MAC/B;AAAA,MACA;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,SAAS;AAAA,MACT,gBAAgB;AAAA,MAChB,gBAAgB;AAAA,MAChB;AAAA,MACA,OAAO,eAAe,QAAQ,IAAI,UAAU;AAAA,IAC9C;AAAA,EACF;AACF;","names":["readFile","access","join","parseYaml","join","join","readFile","access","join","join","access","readFile","parseYaml"]}
|
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getColumnsQuery,
|
|
3
|
+
getDescribeQuery,
|
|
4
|
+
getTablesQuery
|
|
5
|
+
} from "./chunk-EHM6AMMA.js";
|
|
6
|
+
import {
|
|
7
|
+
createConnector,
|
|
8
|
+
resolveConnection
|
|
9
|
+
} from "./chunk-45SP26I2.js";
|
|
10
|
+
import {
|
|
11
|
+
detail,
|
|
12
|
+
error,
|
|
13
|
+
info,
|
|
14
|
+
spinner,
|
|
15
|
+
success,
|
|
16
|
+
warning
|
|
17
|
+
} from "./chunk-HJVVHYVN.js";
|
|
18
|
+
import "./chunk-UDRJFEJU.js";
|
|
19
|
+
import "./chunk-23E6YT4S.js";
|
|
20
|
+
import {
|
|
21
|
+
generateCatalogJson,
|
|
22
|
+
generateCatalogMd,
|
|
23
|
+
generateWarehouseSection
|
|
24
|
+
} from "./chunk-WYS4ULBE.js";
|
|
25
|
+
import "./chunk-DGUM43GV.js";
|
|
26
|
+
|
|
27
|
+
// src/commands/warehouse-sync.ts
|
|
28
|
+
import { readFile, writeFile, mkdir } from "fs/promises";
|
|
29
|
+
import { join } from "path";
|
|
30
|
+
function groupColumnsByTable(columns) {
|
|
31
|
+
const map = /* @__PURE__ */ new Map();
|
|
32
|
+
for (const col of columns) {
|
|
33
|
+
const key = `${col.schema}.${col.table}`;
|
|
34
|
+
const existing = map.get(key);
|
|
35
|
+
if (existing) {
|
|
36
|
+
existing.push(col);
|
|
37
|
+
} else {
|
|
38
|
+
map.set(key, [col]);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return map;
|
|
42
|
+
}
|
|
43
|
+
function diffSyncState(current, prev) {
|
|
44
|
+
const newTables = [];
|
|
45
|
+
const changedTables = [];
|
|
46
|
+
const unchangedTables = [];
|
|
47
|
+
const removedTables = [];
|
|
48
|
+
if (!prev) {
|
|
49
|
+
return {
|
|
50
|
+
newTables: current.map((t) => t.qualifiedName),
|
|
51
|
+
changedTables: [],
|
|
52
|
+
unchangedTables: [],
|
|
53
|
+
removedTables: []
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
const currentNames = new Set(current.map((t) => t.qualifiedName));
|
|
57
|
+
for (const table of current) {
|
|
58
|
+
const prevEntry = prev.tables[table.qualifiedName];
|
|
59
|
+
if (!prevEntry) {
|
|
60
|
+
newTables.push(table.qualifiedName);
|
|
61
|
+
} else if (prevEntry.columnCount !== table.columnCount) {
|
|
62
|
+
changedTables.push(table.qualifiedName);
|
|
63
|
+
} else {
|
|
64
|
+
unchangedTables.push(table.qualifiedName);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
for (const name of Object.keys(prev.tables)) {
|
|
68
|
+
if (!currentNames.has(name)) {
|
|
69
|
+
removedTables.push(name);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return { newTables, changedTables, unchangedTables, removedTables };
|
|
73
|
+
}
|
|
74
|
+
function buildWarehouseModels(tables, columnMap, samples) {
|
|
75
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
76
|
+
return tables.map((table) => {
|
|
77
|
+
const cols = columnMap.get(table.qualifiedName) || [];
|
|
78
|
+
const sampleRows = samples[table.qualifiedName];
|
|
79
|
+
return {
|
|
80
|
+
name: table.name,
|
|
81
|
+
path: "",
|
|
82
|
+
description: "",
|
|
83
|
+
table: table.qualifiedName,
|
|
84
|
+
tags: ["warehouse"],
|
|
85
|
+
meta: {},
|
|
86
|
+
columns: cols.map((c) => ({
|
|
87
|
+
name: c.name,
|
|
88
|
+
data_type: c.type,
|
|
89
|
+
description: "",
|
|
90
|
+
hints: []
|
|
91
|
+
})),
|
|
92
|
+
yamchartModels: [],
|
|
93
|
+
source: "warehouse",
|
|
94
|
+
tableType: table.type,
|
|
95
|
+
...sampleRows ? { sampleRows } : {},
|
|
96
|
+
lastSynced: now
|
|
97
|
+
};
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
function mergeWithDbtCatalog(existingCatalog, warehouseModels) {
|
|
101
|
+
if (!existingCatalog) {
|
|
102
|
+
return {
|
|
103
|
+
syncedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
104
|
+
source: { type: "warehouse" },
|
|
105
|
+
stats: { modelsIncluded: warehouseModels.length, modelsExcluded: 0 },
|
|
106
|
+
models: warehouseModels
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const warehouseByTable = /* @__PURE__ */ new Map();
|
|
110
|
+
for (const wm of warehouseModels) {
|
|
111
|
+
if (wm.table) {
|
|
112
|
+
warehouseByTable.set(wm.table.toUpperCase(), wm);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const consumedWarehouseTables = /* @__PURE__ */ new Set();
|
|
116
|
+
const mergedModels = [];
|
|
117
|
+
for (const model of existingCatalog.models) {
|
|
118
|
+
if (model.source === "warehouse") {
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const tableKey = (model.table || "").toUpperCase();
|
|
122
|
+
const warehouseMatch = tableKey ? warehouseByTable.get(tableKey) : void 0;
|
|
123
|
+
if (warehouseMatch) {
|
|
124
|
+
const warehouseColMap = /* @__PURE__ */ new Map();
|
|
125
|
+
for (const wc of warehouseMatch.columns) {
|
|
126
|
+
warehouseColMap.set(wc.name.toLowerCase(), wc);
|
|
127
|
+
}
|
|
128
|
+
const updatedColumns = model.columns.map((col) => {
|
|
129
|
+
const wc = warehouseColMap.get(col.name.toLowerCase());
|
|
130
|
+
if (wc && !col.data_type && wc.data_type) {
|
|
131
|
+
return { ...col, data_type: wc.data_type };
|
|
132
|
+
}
|
|
133
|
+
return col;
|
|
134
|
+
});
|
|
135
|
+
const existingColNames = new Set(model.columns.map((c) => c.name.toLowerCase()));
|
|
136
|
+
for (const wc of warehouseMatch.columns) {
|
|
137
|
+
if (!existingColNames.has(wc.name.toLowerCase())) {
|
|
138
|
+
updatedColumns.push(wc);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
mergedModels.push({ ...model, columns: updatedColumns });
|
|
142
|
+
consumedWarehouseTables.add(tableKey);
|
|
143
|
+
} else {
|
|
144
|
+
mergedModels.push(model);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const wm of warehouseModels) {
|
|
148
|
+
const tableKey = (wm.table || "").toUpperCase();
|
|
149
|
+
if (!consumedWarehouseTables.has(tableKey)) {
|
|
150
|
+
mergedModels.push(wm);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return {
|
|
154
|
+
...existingCatalog,
|
|
155
|
+
syncedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
156
|
+
models: mergedModels
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
async function syncWarehouse(projectDir, connector, connectionType, connectionName, options) {
|
|
160
|
+
const start = performance.now();
|
|
161
|
+
const errors = [];
|
|
162
|
+
const tablesQuery = getTablesQuery(connectionType, {
|
|
163
|
+
database: options?.database
|
|
164
|
+
});
|
|
165
|
+
const tablesResult = await connector.execute(tablesQuery.sql);
|
|
166
|
+
let normalizedTables = tablesQuery.normalize(tablesResult.rows);
|
|
167
|
+
if (options?.schemas && options.schemas.length > 0) {
|
|
168
|
+
const schemaSet = new Set(options.schemas.map((s) => s.toUpperCase()));
|
|
169
|
+
normalizedTables = normalizedTables.filter((t) => schemaSet.has(t.schema.toUpperCase()));
|
|
170
|
+
}
|
|
171
|
+
let allColumns = [];
|
|
172
|
+
if (connectionType === "sqlite") {
|
|
173
|
+
for (const table of normalizedTables) {
|
|
174
|
+
try {
|
|
175
|
+
const describeQuery = getDescribeQuery("sqlite", table.name);
|
|
176
|
+
const descResult = await connector.execute(describeQuery.sql);
|
|
177
|
+
const cols = describeQuery.normalize(descResult.rows);
|
|
178
|
+
for (const col of cols) {
|
|
179
|
+
allColumns.push({
|
|
180
|
+
schema: table.schema,
|
|
181
|
+
table: table.name,
|
|
182
|
+
name: col.name,
|
|
183
|
+
type: col.type,
|
|
184
|
+
nullable: col.nullable
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
} catch (err) {
|
|
188
|
+
errors.push(`Failed to describe ${table.name}: ${err instanceof Error ? err.message : String(err)}`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
} else {
|
|
192
|
+
const columnsQuery = getColumnsQuery(connectionType, {
|
|
193
|
+
database: options?.database
|
|
194
|
+
});
|
|
195
|
+
const columnsResult = await connector.execute(columnsQuery.sql);
|
|
196
|
+
allColumns = columnsQuery.normalize(columnsResult.rows);
|
|
197
|
+
if (options?.schemas && options.schemas.length > 0) {
|
|
198
|
+
const schemaSet = new Set(options.schemas.map((s) => s.toUpperCase()));
|
|
199
|
+
allColumns = allColumns.filter((c) => schemaSet.has(c.schema.toUpperCase()));
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const columnMap = groupColumnsByTable(allColumns);
|
|
203
|
+
const warehouseTables = normalizedTables.map((t) => {
|
|
204
|
+
const qualifiedName = t.schema ? `${t.schema}.${t.name}` : t.name;
|
|
205
|
+
return {
|
|
206
|
+
qualifiedName,
|
|
207
|
+
schema: t.schema,
|
|
208
|
+
name: t.name,
|
|
209
|
+
type: t.type,
|
|
210
|
+
columnCount: columnMap.get(qualifiedName)?.length || 0
|
|
211
|
+
};
|
|
212
|
+
});
|
|
213
|
+
let prevState = null;
|
|
214
|
+
if (!options?.full) {
|
|
215
|
+
try {
|
|
216
|
+
const raw = await readFile(join(projectDir, ".yamchart", "warehouse-sync.json"), "utf-8");
|
|
217
|
+
prevState = JSON.parse(raw);
|
|
218
|
+
} catch {
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
const diff = diffSyncState(warehouseTables, prevState);
|
|
222
|
+
const samples = {};
|
|
223
|
+
const tablesToSample = options?.skipSamples ? [] : [...diff.newTables, ...diff.changedTables];
|
|
224
|
+
for (let i = 0; i < tablesToSample.length; i++) {
|
|
225
|
+
const tableName = tablesToSample[i];
|
|
226
|
+
options?.onProgress?.(tableName, i + 1, tablesToSample.length);
|
|
227
|
+
try {
|
|
228
|
+
const sampleSql = `SELECT * FROM ${tableName} LIMIT 3`;
|
|
229
|
+
const samplePromise = connector.execute(sampleSql);
|
|
230
|
+
const timeoutPromise = new Promise(
|
|
231
|
+
(_, reject) => setTimeout(() => reject(new Error(`Timeout sampling ${tableName} (5s)`)), 5e3)
|
|
232
|
+
);
|
|
233
|
+
const result = await Promise.race([samplePromise, timeoutPromise]);
|
|
234
|
+
samples[tableName] = result.rows;
|
|
235
|
+
} catch (err) {
|
|
236
|
+
errors.push(`Failed to sample ${tableName}: ${err instanceof Error ? err.message : String(err)}`);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
let existingCatalog = null;
|
|
240
|
+
try {
|
|
241
|
+
const raw = await readFile(join(projectDir, ".yamchart", "catalog.json"), "utf-8");
|
|
242
|
+
existingCatalog = JSON.parse(raw);
|
|
243
|
+
} catch {
|
|
244
|
+
}
|
|
245
|
+
if (existingCatalog && !options?.skipSamples) {
|
|
246
|
+
for (const tableName of diff.unchangedTables) {
|
|
247
|
+
const existingModel = existingCatalog.models.find(
|
|
248
|
+
(m) => m.table?.toUpperCase() === tableName.toUpperCase() && m.source === "warehouse"
|
|
249
|
+
);
|
|
250
|
+
if (existingModel?.sampleRows) {
|
|
251
|
+
samples[tableName] = existingModel.sampleRows;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
const warehouseModels = buildWarehouseModels(warehouseTables, columnMap, samples);
|
|
256
|
+
const merged = mergeWithDbtCatalog(existingCatalog, warehouseModels);
|
|
257
|
+
await mkdir(join(projectDir, ".yamchart"), { recursive: true });
|
|
258
|
+
await writeFile(join(projectDir, ".yamchart", "catalog.json"), generateCatalogJson(merged));
|
|
259
|
+
const catalogMd = generateCatalogMd(merged);
|
|
260
|
+
const syncDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
|
|
261
|
+
const warehouseSection = generateWarehouseSection(merged.models, connectionName, syncDate);
|
|
262
|
+
const fullMd = warehouseSection ? catalogMd + "\n" + warehouseSection : catalogMd;
|
|
263
|
+
await writeFile(join(projectDir, ".yamchart", "catalog.md"), fullMd);
|
|
264
|
+
const syncState = {
|
|
265
|
+
syncedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
266
|
+
connection: connectionName,
|
|
267
|
+
schemas: options?.schemas || [],
|
|
268
|
+
skipSamples: options?.skipSamples || false,
|
|
269
|
+
tables: Object.fromEntries(
|
|
270
|
+
warehouseTables.map((t) => [t.qualifiedName, { columnCount: t.columnCount }])
|
|
271
|
+
)
|
|
272
|
+
};
|
|
273
|
+
await writeFile(join(projectDir, ".yamchart", "warehouse-sync.json"), JSON.stringify(syncState, null, 2));
|
|
274
|
+
const durationMs = Math.round(performance.now() - start);
|
|
275
|
+
return {
|
|
276
|
+
tablesTotal: warehouseTables.length,
|
|
277
|
+
tablesNew: diff.newTables.length,
|
|
278
|
+
tablesChanged: diff.changedTables.length,
|
|
279
|
+
tablesUnchanged: diff.unchangedTables.length,
|
|
280
|
+
tablesRemoved: diff.removedTables.length,
|
|
281
|
+
errors,
|
|
282
|
+
durationMs
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// src/commands/sync-warehouse.ts
|
|
287
|
+
import { readFile as readFile2 } from "fs/promises";
|
|
288
|
+
import { join as join2 } from "path";
|
|
289
|
+
async function runSyncWarehouse(projectDir, options) {
|
|
290
|
+
let connectionOverride = options.connection;
|
|
291
|
+
let schemasOverride = options.schema;
|
|
292
|
+
if (options.refresh) {
|
|
293
|
+
try {
|
|
294
|
+
const raw = await readFile2(join2(projectDir, ".yamchart", "warehouse-sync.json"), "utf-8");
|
|
295
|
+
const prevState = JSON.parse(raw);
|
|
296
|
+
connectionOverride = connectionOverride || prevState.connection;
|
|
297
|
+
if (!schemasOverride && prevState.schemas.length > 0) {
|
|
298
|
+
schemasOverride = prevState.schemas.join(",");
|
|
299
|
+
}
|
|
300
|
+
if (!options.json) {
|
|
301
|
+
info(`Re-syncing from ${prevState.connection} (${prevState.schemas.join(", ") || "all schemas"})`);
|
|
302
|
+
}
|
|
303
|
+
} catch {
|
|
304
|
+
error("No previous sync state found. Run sync-warehouse without --refresh first.");
|
|
305
|
+
process.exit(1);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
const connection = await resolveConnection(projectDir, connectionOverride);
|
|
309
|
+
const connector = createConnector(connection, projectDir);
|
|
310
|
+
const schemas = schemasOverride ? schemasOverride.split(",").map((s) => s.trim()) : void 0;
|
|
311
|
+
const spin = options.json ? null : spinner(`Connecting to ${connection.name} ...`);
|
|
312
|
+
try {
|
|
313
|
+
await connector.connect();
|
|
314
|
+
if (spin) {
|
|
315
|
+
spin.text = "Fetching table metadata...";
|
|
316
|
+
}
|
|
317
|
+
const result = await syncWarehouse(
|
|
318
|
+
projectDir,
|
|
319
|
+
connector,
|
|
320
|
+
connection.type,
|
|
321
|
+
connection.name,
|
|
322
|
+
{
|
|
323
|
+
schemas,
|
|
324
|
+
database: options.database,
|
|
325
|
+
skipSamples: options.skipSamples,
|
|
326
|
+
full: options.full,
|
|
327
|
+
onProgress: spin ? (table, index, total) => {
|
|
328
|
+
spin.text = `Sampling tables... ${index}/${total} (${table})`;
|
|
329
|
+
} : void 0
|
|
330
|
+
}
|
|
331
|
+
);
|
|
332
|
+
if (spin) {
|
|
333
|
+
spin.stop();
|
|
334
|
+
}
|
|
335
|
+
if (options.json) {
|
|
336
|
+
console.log(JSON.stringify(result, null, 2));
|
|
337
|
+
} else {
|
|
338
|
+
const parts = [
|
|
339
|
+
`${result.tablesNew} new`,
|
|
340
|
+
`${result.tablesChanged} changed`,
|
|
341
|
+
`${result.tablesUnchanged} unchanged`
|
|
342
|
+
];
|
|
343
|
+
if (result.tablesRemoved > 0) {
|
|
344
|
+
parts.push(`${result.tablesRemoved} removed`);
|
|
345
|
+
}
|
|
346
|
+
success(
|
|
347
|
+
`Synced ${result.tablesTotal} tables (${parts.join(", ")}) in ${(result.durationMs / 1e3).toFixed(1)}s`
|
|
348
|
+
);
|
|
349
|
+
detail("Written to .yamchart/catalog.json");
|
|
350
|
+
if (result.errors.length > 0) {
|
|
351
|
+
warning(`${result.errors.length} table(s) had sampling errors:`);
|
|
352
|
+
for (const err of result.errors.slice(0, 5)) {
|
|
353
|
+
detail(err);
|
|
354
|
+
}
|
|
355
|
+
if (result.errors.length > 5) {
|
|
356
|
+
detail(`... and ${result.errors.length - 5} more`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
} finally {
|
|
361
|
+
await connector.disconnect();
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
export {
|
|
365
|
+
runSyncWarehouse
|
|
366
|
+
};
|
|
367
|
+
//# sourceMappingURL=sync-warehouse-WPNGI6YP.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/commands/warehouse-sync.ts","../src/commands/sync-warehouse.ts"],"sourcesContent":["/**\n * Core warehouse sync logic.\n *\n * Pure functions for diffing, grouping, building models, and merging catalogs,\n * plus an async orchestrator (syncWarehouse) that does I/O.\n */\n\nimport { readFile, writeFile, mkdir } from 'fs/promises';\nimport { join } from 'path';\nimport type { Connector } from '@yamchart/query';\nimport type { NormalizedColumnWithTable } from './introspection.js';\nimport { getTablesQuery, getColumnsQuery, getDescribeQuery } from './introspection.js';\nimport type { CatalogModel, CatalogData } from '../dbt/catalog.js';\nimport { generateCatalogMd, generateCatalogJson, generateWarehouseSection } from '../dbt/catalog.js';\n\n// ---------------------------------------------------------------------------\n// Interfaces\n// ---------------------------------------------------------------------------\n\nexport interface SyncState {\n syncedAt: string;\n connection: string;\n schemas: string[];\n skipSamples: boolean;\n tables: Record<string, { columnCount: number; lastAltered?: string }>;\n}\n\nexport interface WarehouseTable {\n qualifiedName: string; // \"SCHEMA.TABLE\"\n schema: string;\n name: string;\n type: string; // \"TABLE\" | \"VIEW\"\n columnCount: number;\n}\n\nexport interface SyncDiff {\n newTables: string[];\n changedTables: string[];\n unchangedTables: string[];\n removedTables: string[];\n}\n\nexport interface SyncWarehouseResult {\n tablesTotal: number;\n tablesNew: number;\n tablesChanged: number;\n tablesUnchanged: number;\n tablesRemoved: number;\n errors: string[];\n durationMs: number;\n}\n\nexport interface SyncWarehouseOptions {\n schemas?: string[];\n database?: string;\n skipSamples?: boolean;\n full?: boolean;\n onProgress?: (table: string, index: number, total: number) => void;\n}\n\n// ---------------------------------------------------------------------------\n// Pure functions\n// ---------------------------------------------------------------------------\n\n/**\n * Group a flat column list into a Map keyed by \"SCHEMA.TABLE\".\n */\nexport function groupColumnsByTable(\n columns: NormalizedColumnWithTable[],\n): Map<string, NormalizedColumnWithTable[]> {\n const map = new Map<string, NormalizedColumnWithTable[]>();\n for (const col of columns) {\n const key = `${col.schema}.${col.table}`;\n const existing = map.get(key);\n if (existing) {\n existing.push(col);\n } else {\n map.set(key, [col]);\n }\n }\n return map;\n}\n\n/**\n * Compare current warehouse tables against previous sync state.\n * All tables are \"new\" when prev is null.\n * Detects changed by column count mismatch.\n * Detects removed tables (in prev but not in current).\n */\nexport function diffSyncState(\n current: WarehouseTable[],\n prev: SyncState | null,\n): SyncDiff {\n const newTables: string[] = [];\n const changedTables: string[] = [];\n const unchangedTables: string[] = [];\n const removedTables: string[] = [];\n\n if (!prev) {\n return {\n newTables: current.map((t) => t.qualifiedName),\n changedTables: [],\n unchangedTables: [],\n removedTables: [],\n };\n }\n\n const currentNames = new Set(current.map((t) => t.qualifiedName));\n\n for (const table of current) {\n const prevEntry = prev.tables[table.qualifiedName];\n if (!prevEntry) {\n newTables.push(table.qualifiedName);\n } else if (prevEntry.columnCount !== table.columnCount) {\n changedTables.push(table.qualifiedName);\n } else {\n unchangedTables.push(table.qualifiedName);\n }\n }\n\n // Detect removed tables\n for (const name of Object.keys(prev.tables)) {\n if (!currentNames.has(name)) {\n removedTables.push(name);\n }\n }\n\n return { newTables, changedTables, unchangedTables, removedTables };\n}\n\n/**\n * Build CatalogModel entries from warehouse metadata.\n */\nexport function buildWarehouseModels(\n tables: WarehouseTable[],\n columnMap: Map<string, NormalizedColumnWithTable[]>,\n samples: Record<string, Record<string, unknown>[]>,\n): CatalogModel[] {\n const now = new Date().toISOString();\n\n return tables.map((table) => {\n const cols = columnMap.get(table.qualifiedName) || [];\n const sampleRows = samples[table.qualifiedName];\n\n return {\n name: table.name,\n path: '',\n description: '',\n table: table.qualifiedName,\n tags: ['warehouse'],\n meta: {},\n columns: cols.map((c) => ({\n name: c.name,\n data_type: c.type,\n description: '',\n hints: [],\n })),\n yamchartModels: [],\n source: 'warehouse' as const,\n tableType: table.type as 'TABLE' | 'VIEW',\n ...(sampleRows ? { sampleRows } : {}),\n lastSynced: now,\n };\n });\n}\n\n/**\n * Merge warehouse models into an existing dbt catalog.\n *\n * 1. If no existing catalog, create new with warehouseModels.\n * 2. Remove stale warehouse entries from existing.\n * 3. For tables in both dbt and warehouse (matched by `table` field, case-insensitive):\n * - dbt entry wins (keep it)\n * - Backfill data_type from warehouse columns where missing\n * - Add any warehouse columns that don't exist in the dbt entry\n * 4. Append remaining warehouse-only models.\n * 5. Return updated CatalogData.\n */\nexport function mergeWithDbtCatalog(\n existingCatalog: CatalogData | null,\n warehouseModels: CatalogModel[],\n): CatalogData {\n if (!existingCatalog) {\n return {\n syncedAt: new Date().toISOString(),\n source: { type: 'warehouse' },\n stats: { modelsIncluded: warehouseModels.length, modelsExcluded: 0 },\n models: warehouseModels,\n };\n }\n\n // Build a lookup of warehouse models by normalized table name\n const warehouseByTable = new Map<string, CatalogModel>();\n for (const wm of warehouseModels) {\n if (wm.table) {\n warehouseByTable.set(wm.table.toUpperCase(), wm);\n }\n }\n\n // Track which warehouse models were consumed by dbt merge\n const consumedWarehouseTables = new Set<string>();\n\n // Process existing catalog: keep non-warehouse entries, merge dbt+warehouse matches\n const mergedModels: CatalogModel[] = [];\n\n for (const model of existingCatalog.models) {\n // Remove stale warehouse entries — they'll be replaced by current warehouseModels\n if (model.source === 'warehouse') {\n continue;\n }\n\n // Check if this dbt model matches a warehouse table\n const tableKey = (model.table || '').toUpperCase();\n const warehouseMatch = tableKey ? warehouseByTable.get(tableKey) : undefined;\n\n if (warehouseMatch) {\n // dbt entry wins — backfill data_type and add missing columns\n const warehouseColMap = new Map<string, { data_type?: string; description: string; hints: string[] }>();\n for (const wc of warehouseMatch.columns) {\n warehouseColMap.set(wc.name.toLowerCase(), wc);\n }\n\n // Backfill data_type on existing dbt columns\n const updatedColumns = model.columns.map((col) => {\n const wc = warehouseColMap.get(col.name.toLowerCase());\n if (wc && !col.data_type && wc.data_type) {\n return { ...col, data_type: wc.data_type };\n }\n return col;\n });\n\n // Add warehouse columns not present in dbt entry\n const existingColNames = new Set(model.columns.map((c) => c.name.toLowerCase()));\n for (const wc of warehouseMatch.columns) {\n if (!existingColNames.has(wc.name.toLowerCase())) {\n updatedColumns.push(wc);\n }\n }\n\n mergedModels.push({ ...model, columns: updatedColumns });\n consumedWarehouseTables.add(tableKey);\n } else {\n mergedModels.push(model);\n }\n }\n\n // Append warehouse-only models (not consumed by dbt merge)\n for (const wm of warehouseModels) {\n const tableKey = (wm.table || '').toUpperCase();\n if (!consumedWarehouseTables.has(tableKey)) {\n mergedModels.push(wm);\n }\n }\n\n return {\n ...existingCatalog,\n syncedAt: new Date().toISOString(),\n models: mergedModels,\n };\n}\n\n// ---------------------------------------------------------------------------\n// Async orchestrator\n// ---------------------------------------------------------------------------\n\n/**\n * Main warehouse sync orchestrator.\n *\n * 1. Fetch tables and columns from the warehouse.\n * 2. Diff against previous sync state.\n * 3. Sample new/changed tables (unless skipSamples).\n * 4. Build warehouse CatalogModel entries.\n * 5. Merge with existing catalog, write files.\n */\nexport async function syncWarehouse(\n projectDir: string,\n connector: Connector,\n connectionType: string,\n connectionName: string,\n options?: SyncWarehouseOptions,\n): Promise<SyncWarehouseResult> {\n const start = performance.now();\n const errors: string[] = [];\n\n // 1. Fetch tables\n const tablesQuery = getTablesQuery(connectionType, {\n database: options?.database,\n });\n const tablesResult = await connector.execute(tablesQuery.sql);\n let normalizedTables = tablesQuery.normalize(tablesResult.rows);\n\n // Filter by schemas if specified\n if (options?.schemas && options.schemas.length > 0) {\n const schemaSet = new Set(options.schemas.map((s) => s.toUpperCase()));\n normalizedTables = normalizedTables.filter((t) => schemaSet.has(t.schema.toUpperCase()));\n }\n\n // 2. Fetch columns\n let allColumns: NormalizedColumnWithTable[] = [];\n\n if (connectionType === 'sqlite') {\n // SQLite: describe each table individually\n for (const table of normalizedTables) {\n try {\n const describeQuery = getDescribeQuery('sqlite', table.name);\n const descResult = await connector.execute(describeQuery.sql);\n const cols = describeQuery.normalize(descResult.rows);\n for (const col of cols) {\n allColumns.push({\n schema: table.schema,\n table: table.name,\n name: col.name,\n type: col.type,\n nullable: col.nullable,\n });\n }\n } catch (err) {\n errors.push(`Failed to describe ${table.name}: ${err instanceof Error ? err.message : String(err)}`);\n }\n }\n } else {\n const columnsQuery = getColumnsQuery(connectionType, {\n database: options?.database,\n });\n const columnsResult = await connector.execute(columnsQuery.sql);\n allColumns = columnsQuery.normalize(columnsResult.rows);\n\n // Filter columns by schemas if specified\n if (options?.schemas && options.schemas.length > 0) {\n const schemaSet = new Set(options.schemas.map((s) => s.toUpperCase()));\n allColumns = allColumns.filter((c) => schemaSet.has(c.schema.toUpperCase()));\n }\n }\n\n // Group columns by table\n const columnMap = groupColumnsByTable(allColumns);\n\n // Build WarehouseTable entries with column counts\n const warehouseTables: WarehouseTable[] = normalizedTables.map((t) => {\n const qualifiedName = t.schema ? `${t.schema}.${t.name}` : t.name;\n return {\n qualifiedName,\n schema: t.schema,\n name: t.name,\n type: t.type,\n columnCount: columnMap.get(qualifiedName)?.length || 0,\n };\n });\n\n // 3. Load previous sync state, compute diff\n let prevState: SyncState | null = null;\n if (!options?.full) {\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), 'utf-8');\n prevState = JSON.parse(raw);\n } catch {\n /* no previous state */\n }\n }\n\n const diff = diffSyncState(warehouseTables, prevState);\n\n // 4. Sample new/changed tables (unless skipSamples)\n const samples: Record<string, Record<string, unknown>[]> = {};\n const tablesToSample = options?.skipSamples\n ? []\n : [...diff.newTables, ...diff.changedTables];\n\n for (let i = 0; i < tablesToSample.length; i++) {\n const tableName = tablesToSample[i]!;\n options?.onProgress?.(tableName, i + 1, tablesToSample.length);\n\n try {\n const sampleSql = `SELECT * FROM ${tableName} LIMIT 3`;\n const samplePromise = connector.execute(sampleSql);\n const timeoutPromise = new Promise<never>((_, reject) =>\n setTimeout(() => reject(new Error(`Timeout sampling ${tableName} (5s)`)), 5000),\n );\n const result = await Promise.race([samplePromise, timeoutPromise]);\n samples[tableName] = result.rows;\n } catch (err) {\n errors.push(`Failed to sample ${tableName}: ${err instanceof Error ? err.message : String(err)}`);\n }\n }\n\n // 5. For unchanged tables: carry forward sampleRows from existing catalog\n let existingCatalog: CatalogData | null = null;\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'catalog.json'), 'utf-8');\n existingCatalog = JSON.parse(raw);\n } catch {\n /* no existing catalog */\n }\n\n if (existingCatalog && !options?.skipSamples) {\n for (const tableName of diff.unchangedTables) {\n const existingModel = existingCatalog.models.find(\n (m) => m.table?.toUpperCase() === tableName.toUpperCase() && m.source === 'warehouse',\n );\n if (existingModel?.sampleRows) {\n samples[tableName] = existingModel.sampleRows;\n }\n }\n }\n\n // 6. Build warehouse CatalogModel entries\n const warehouseModels = buildWarehouseModels(warehouseTables, columnMap, samples);\n\n // 7. Merge with existing catalog, write files\n const merged = mergeWithDbtCatalog(existingCatalog, warehouseModels);\n\n await mkdir(join(projectDir, '.yamchart'), { recursive: true });\n await writeFile(join(projectDir, '.yamchart', 'catalog.json'), generateCatalogJson(merged));\n\n const catalogMd = generateCatalogMd(merged);\n const syncDate = new Date().toISOString().split('T')[0]!;\n const warehouseSection = generateWarehouseSection(merged.models, connectionName, syncDate);\n const fullMd = warehouseSection ? catalogMd + '\\n' + warehouseSection : catalogMd;\n await writeFile(join(projectDir, '.yamchart', 'catalog.md'), fullMd);\n\n // Write sync state\n const syncState: SyncState = {\n syncedAt: new Date().toISOString(),\n connection: connectionName,\n schemas: options?.schemas || [],\n skipSamples: options?.skipSamples || false,\n tables: Object.fromEntries(\n warehouseTables.map((t) => [t.qualifiedName, { columnCount: t.columnCount }]),\n ),\n };\n await writeFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), JSON.stringify(syncState, null, 2));\n\n const durationMs = Math.round(performance.now() - start);\n\n return {\n tablesTotal: warehouseTables.length,\n tablesNew: diff.newTables.length,\n tablesChanged: diff.changedTables.length,\n tablesUnchanged: diff.unchangedTables.length,\n tablesRemoved: diff.removedTables.length,\n errors,\n durationMs,\n };\n}\n","/**\n * CLI handler for `yamchart sync-warehouse`.\n *\n * Thin wrapper: resolves connection, creates connector, delegates to syncWarehouse().\n */\n\nimport * as output from '../utils/output.js';\nimport { resolveConnection, createConnector } from './connection-utils.js';\nimport { syncWarehouse } from './warehouse-sync.js';\nimport { readFile } from 'fs/promises';\nimport { join } from 'path';\nimport type { SyncState } from './warehouse-sync.js';\n\nexport interface SyncWarehouseCliOptions {\n connection?: string;\n schema?: string; // comma-separated → split into array\n database?: string;\n skipSamples?: boolean;\n refresh?: boolean;\n full?: boolean;\n json?: boolean;\n}\n\nexport async function runSyncWarehouse(\n projectDir: string,\n options: SyncWarehouseCliOptions,\n): Promise<void> {\n // Handle --refresh: load saved connection/schemas from previous sync state\n let connectionOverride = options.connection;\n let schemasOverride = options.schema;\n\n if (options.refresh) {\n try {\n const raw = await readFile(join(projectDir, '.yamchart', 'warehouse-sync.json'), 'utf-8');\n const prevState: SyncState = JSON.parse(raw);\n connectionOverride = connectionOverride || prevState.connection;\n if (!schemasOverride && prevState.schemas.length > 0) {\n schemasOverride = prevState.schemas.join(',');\n }\n if (!options.json) {\n output.info(`Re-syncing from ${prevState.connection} (${prevState.schemas.join(', ') || 'all schemas'})`);\n }\n } catch {\n output.error('No previous sync state found. Run sync-warehouse without --refresh first.');\n process.exit(1);\n }\n }\n\n // Resolve connection\n const connection = await resolveConnection(projectDir, connectionOverride);\n const connector = createConnector(connection, projectDir);\n\n const schemas = schemasOverride\n ? schemasOverride.split(',').map((s) => s.trim())\n : undefined;\n\n const spin = options.json ? null : output.spinner(`Connecting to ${connection.name} ...`);\n\n try {\n await connector.connect();\n\n if (spin) {\n spin.text = 'Fetching table metadata...';\n }\n\n const result = await syncWarehouse(\n projectDir,\n connector,\n connection.type,\n connection.name,\n {\n schemas,\n database: options.database,\n skipSamples: options.skipSamples,\n full: options.full,\n onProgress: spin\n ? (table, index, total) => {\n spin.text = `Sampling tables... ${index}/${total} (${table})`;\n }\n : undefined,\n },\n );\n\n if (spin) {\n spin.stop();\n }\n\n if (options.json) {\n console.log(JSON.stringify(result, null, 2));\n } else {\n const parts = [\n `${result.tablesNew} new`,\n `${result.tablesChanged} changed`,\n `${result.tablesUnchanged} unchanged`,\n ];\n if (result.tablesRemoved > 0) {\n parts.push(`${result.tablesRemoved} removed`);\n }\n output.success(\n `Synced ${result.tablesTotal} tables (${parts.join(', ')}) in ${(result.durationMs / 1000).toFixed(1)}s`,\n );\n output.detail('Written to .yamchart/catalog.json');\n\n if (result.errors.length > 0) {\n output.warning(`${result.errors.length} table(s) had sampling errors:`);\n for (const err of result.errors.slice(0, 5)) {\n output.detail(err);\n }\n if (result.errors.length > 5) {\n output.detail(`... and ${result.errors.length - 5} more`);\n }\n }\n }\n } finally {\n await connector.disconnect();\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA,SAAS,UAAU,WAAW,aAAa;AAC3C,SAAS,YAAY;AA2Dd,SAAS,oBACd,SAC0C;AAC1C,QAAM,MAAM,oBAAI,IAAyC;AACzD,aAAW,OAAO,SAAS;AACzB,UAAM,MAAM,GAAG,IAAI,MAAM,IAAI,IAAI,KAAK;AACtC,UAAM,WAAW,IAAI,IAAI,GAAG;AAC5B,QAAI,UAAU;AACZ,eAAS,KAAK,GAAG;AAAA,IACnB,OAAO;AACL,UAAI,IAAI,KAAK,CAAC,GAAG,CAAC;AAAA,IACpB;AAAA,EACF;AACA,SAAO;AACT;AAQO,SAAS,cACd,SACA,MACU;AACV,QAAM,YAAsB,CAAC;AAC7B,QAAM,gBAA0B,CAAC;AACjC,QAAM,kBAA4B,CAAC;AACnC,QAAM,gBAA0B,CAAC;AAEjC,MAAI,CAAC,MAAM;AACT,WAAO;AAAA,MACL,WAAW,QAAQ,IAAI,CAAC,MAAM,EAAE,aAAa;AAAA,MAC7C,eAAe,CAAC;AAAA,MAChB,iBAAiB,CAAC;AAAA,MAClB,eAAe,CAAC;AAAA,IAClB;AAAA,EACF;AAEA,QAAM,eAAe,IAAI,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC;AAEhE,aAAW,SAAS,SAAS;AAC3B,UAAM,YAAY,KAAK,OAAO,MAAM,aAAa;AACjD,QAAI,CAAC,WAAW;AACd,gBAAU,KAAK,MAAM,aAAa;AAAA,IACpC,WAAW,UAAU,gBAAgB,MAAM,aAAa;AACtD,oBAAc,KAAK,MAAM,aAAa;AAAA,IACxC,OAAO;AACL,sBAAgB,KAAK,MAAM,aAAa;AAAA,IAC1C;AAAA,EACF;AAGA,aAAW,QAAQ,OAAO,KAAK,KAAK,MAAM,GAAG;AAC3C,QAAI,CAAC,aAAa,IAAI,IAAI,GAAG;AAC3B,oBAAc,KAAK,IAAI;AAAA,IACzB;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,eAAe,iBAAiB,cAAc;AACpE;AAKO,SAAS,qBACd,QACA,WACA,SACgB;AAChB,QAAM,OAAM,oBAAI,KAAK,GAAE,YAAY;AAEnC,SAAO,OAAO,IAAI,CAAC,UAAU;AAC3B,UAAM,OAAO,UAAU,IAAI,MAAM,aAAa,KAAK,CAAC;AACpD,UAAM,aAAa,QAAQ,MAAM,aAAa;AAE9C,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,MAAM;AAAA,MACN,aAAa;AAAA,MACb,OAAO,MAAM;AAAA,MACb,MAAM,CAAC,WAAW;AAAA,MAClB,MAAM,CAAC;AAAA,MACP,SAAS,KAAK,IAAI,CAAC,OAAO;AAAA,QACxB,MAAM,EAAE;AAAA,QACR,WAAW,EAAE;AAAA,QACb,aAAa;AAAA,QACb,OAAO,CAAC;AAAA,MACV,EAAE;AAAA,MACF,gBAAgB,CAAC;AAAA,MACjB,QAAQ;AAAA,MACR,WAAW,MAAM;AAAA,MACjB,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,MACnC,YAAY;AAAA,IACd;AAAA,EACF,CAAC;AACH;AAcO,SAAS,oBACd,iBACA,iBACa;AACb,MAAI,CAAC,iBAAiB;AACpB,WAAO;AAAA,MACL,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,MACjC,QAAQ,EAAE,MAAM,YAAY;AAAA,MAC5B,OAAO,EAAE,gBAAgB,gBAAgB,QAAQ,gBAAgB,EAAE;AAAA,MACnE,QAAQ;AAAA,IACV;AAAA,EACF;AAGA,QAAM,mBAAmB,oBAAI,IAA0B;AACvD,aAAW,MAAM,iBAAiB;AAChC,QAAI,GAAG,OAAO;AACZ,uBAAiB,IAAI,GAAG,MAAM,YAAY,GAAG,EAAE;AAAA,IACjD;AAAA,EACF;AAGA,QAAM,0BAA0B,oBAAI,IAAY;AAGhD,QAAM,eAA+B,CAAC;AAEtC,aAAW,SAAS,gBAAgB,QAAQ;AAE1C,QAAI,MAAM,WAAW,aAAa;AAChC;AAAA,IACF;AAGA,UAAM,YAAY,MAAM,SAAS,IAAI,YAAY;AACjD,UAAM,iBAAiB,WAAW,iBAAiB,IAAI,QAAQ,IAAI;AAEnE,QAAI,gBAAgB;AAElB,YAAM,kBAAkB,oBAAI,IAA0E;AACtG,iBAAW,MAAM,eAAe,SAAS;AACvC,wBAAgB,IAAI,GAAG,KAAK,YAAY,GAAG,EAAE;AAAA,MAC/C;AAGA,YAAM,iBAAiB,MAAM,QAAQ,IAAI,CAAC,QAAQ;AAChD,cAAM,KAAK,gBAAgB,IAAI,IAAI,KAAK,YAAY,CAAC;AACrD,YAAI,MAAM,CAAC,IAAI,aAAa,GAAG,WAAW;AACxC,iBAAO,EAAE,GAAG,KAAK,WAAW,GAAG,UAAU;AAAA,QAC3C;AACA,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,mBAAmB,IAAI,IAAI,MAAM,QAAQ,IAAI,CAAC,MAAM,EAAE,KAAK,YAAY,CAAC,CAAC;AAC/E,iBAAW,MAAM,eAAe,SAAS;AACvC,YAAI,CAAC,iBAAiB,IAAI,GAAG,KAAK,YAAY,CAAC,GAAG;AAChD,yBAAe,KAAK,EAAE;AAAA,QACxB;AAAA,MACF;AAEA,mBAAa,KAAK,EAAE,GAAG,OAAO,SAAS,eAAe,CAAC;AACvD,8BAAwB,IAAI,QAAQ;AAAA,IACtC,OAAO;AACL,mBAAa,KAAK,KAAK;AAAA,IACzB;AAAA,EACF;AAGA,aAAW,MAAM,iBAAiB;AAChC,UAAM,YAAY,GAAG,SAAS,IAAI,YAAY;AAC9C,QAAI,CAAC,wBAAwB,IAAI,QAAQ,GAAG;AAC1C,mBAAa,KAAK,EAAE;AAAA,IACtB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,GAAG;AAAA,IACH,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,IACjC,QAAQ;AAAA,EACV;AACF;AAeA,eAAsB,cACpB,YACA,WACA,gBACA,gBACA,SAC8B;AAC9B,QAAM,QAAQ,YAAY,IAAI;AAC9B,QAAM,SAAmB,CAAC;AAG1B,QAAM,cAAc,eAAe,gBAAgB;AAAA,IACjD,UAAU,SAAS;AAAA,EACrB,CAAC;AACD,QAAM,eAAe,MAAM,UAAU,QAAQ,YAAY,GAAG;AAC5D,MAAI,mBAAmB,YAAY,UAAU,aAAa,IAAI;AAG9D,MAAI,SAAS,WAAW,QAAQ,QAAQ,SAAS,GAAG;AAClD,UAAM,YAAY,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACrE,uBAAmB,iBAAiB,OAAO,CAAC,MAAM,UAAU,IAAI,EAAE,OAAO,YAAY,CAAC,CAAC;AAAA,EACzF;AAGA,MAAI,aAA0C,CAAC;AAE/C,MAAI,mBAAmB,UAAU;AAE/B,eAAW,SAAS,kBAAkB;AACpC,UAAI;AACF,cAAM,gBAAgB,iBAAiB,UAAU,MAAM,IAAI;AAC3D,cAAM,aAAa,MAAM,UAAU,QAAQ,cAAc,GAAG;AAC5D,cAAM,OAAO,cAAc,UAAU,WAAW,IAAI;AACpD,mBAAW,OAAO,MAAM;AACtB,qBAAW,KAAK;AAAA,YACd,QAAQ,MAAM;AAAA,YACd,OAAO,MAAM;AAAA,YACb,MAAM,IAAI;AAAA,YACV,MAAM,IAAI;AAAA,YACV,UAAU,IAAI;AAAA,UAChB,CAAC;AAAA,QACH;AAAA,MACF,SAAS,KAAK;AACZ,eAAO,KAAK,sBAAsB,MAAM,IAAI,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,MACrG;AAAA,IACF;AAAA,EACF,OAAO;AACL,UAAM,eAAe,gBAAgB,gBAAgB;AAAA,MACnD,UAAU,SAAS;AAAA,IACrB,CAAC;AACD,UAAM,gBAAgB,MAAM,UAAU,QAAQ,aAAa,GAAG;AAC9D,iBAAa,aAAa,UAAU,cAAc,IAAI;AAGtD,QAAI,SAAS,WAAW,QAAQ,QAAQ,SAAS,GAAG;AAClD,YAAM,YAAY,IAAI,IAAI,QAAQ,QAAQ,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACrE,mBAAa,WAAW,OAAO,CAAC,MAAM,UAAU,IAAI,EAAE,OAAO,YAAY,CAAC,CAAC;AAAA,IAC7E;AAAA,EACF;AAGA,QAAM,YAAY,oBAAoB,UAAU;AAGhD,QAAM,kBAAoC,iBAAiB,IAAI,CAAC,MAAM;AACpE,UAAM,gBAAgB,EAAE,SAAS,GAAG,EAAE,MAAM,IAAI,EAAE,IAAI,KAAK,EAAE;AAC7D,WAAO;AAAA,MACL;AAAA,MACA,QAAQ,EAAE;AAAA,MACV,MAAM,EAAE;AAAA,MACR,MAAM,EAAE;AAAA,MACR,aAAa,UAAU,IAAI,aAAa,GAAG,UAAU;AAAA,IACvD;AAAA,EACF,CAAC;AAGD,MAAI,YAA8B;AAClC,MAAI,CAAC,SAAS,MAAM;AAClB,QAAI;AACF,YAAM,MAAM,MAAM,SAAS,KAAK,YAAY,aAAa,qBAAqB,GAAG,OAAO;AACxF,kBAAY,KAAK,MAAM,GAAG;AAAA,IAC5B,QAAQ;AAAA,IAER;AAAA,EACF;AAEA,QAAM,OAAO,cAAc,iBAAiB,SAAS;AAGrD,QAAM,UAAqD,CAAC;AAC5D,QAAM,iBAAiB,SAAS,cAC5B,CAAC,IACD,CAAC,GAAG,KAAK,WAAW,GAAG,KAAK,aAAa;AAE7C,WAAS,IAAI,GAAG,IAAI,eAAe,QAAQ,KAAK;AAC9C,UAAM,YAAY,eAAe,CAAC;AAClC,aAAS,aAAa,WAAW,IAAI,GAAG,eAAe,MAAM;AAE7D,QAAI;AACF,YAAM,YAAY,iBAAiB,SAAS;AAC5C,YAAM,gBAAgB,UAAU,QAAQ,SAAS;AACjD,YAAM,iBAAiB,IAAI;AAAA,QAAe,CAAC,GAAG,WAC5C,WAAW,MAAM,OAAO,IAAI,MAAM,oBAAoB,SAAS,OAAO,CAAC,GAAG,GAAI;AAAA,MAChF;AACA,YAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,eAAe,cAAc,CAAC;AACjE,cAAQ,SAAS,IAAI,OAAO;AAAA,IAC9B,SAAS,KAAK;AACZ,aAAO,KAAK,oBAAoB,SAAS,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,IAClG;AAAA,EACF;AAGA,MAAI,kBAAsC;AAC1C,MAAI;AACF,UAAM,MAAM,MAAM,SAAS,KAAK,YAAY,aAAa,cAAc,GAAG,OAAO;AACjF,sBAAkB,KAAK,MAAM,GAAG;AAAA,EAClC,QAAQ;AAAA,EAER;AAEA,MAAI,mBAAmB,CAAC,SAAS,aAAa;AAC5C,eAAW,aAAa,KAAK,iBAAiB;AAC5C,YAAM,gBAAgB,gBAAgB,OAAO;AAAA,QAC3C,CAAC,MAAM,EAAE,OAAO,YAAY,MAAM,UAAU,YAAY,KAAK,EAAE,WAAW;AAAA,MAC5E;AACA,UAAI,eAAe,YAAY;AAC7B,gBAAQ,SAAS,IAAI,cAAc;AAAA,MACrC;AAAA,IACF;AAAA,EACF;AAGA,QAAM,kBAAkB,qBAAqB,iBAAiB,WAAW,OAAO;AAGhF,QAAM,SAAS,oBAAoB,iBAAiB,eAAe;AAEnE,QAAM,MAAM,KAAK,YAAY,WAAW,GAAG,EAAE,WAAW,KAAK,CAAC;AAC9D,QAAM,UAAU,KAAK,YAAY,aAAa,cAAc,GAAG,oBAAoB,MAAM,CAAC;AAE1F,QAAM,YAAY,kBAAkB,MAAM;AAC1C,QAAM,YAAW,oBAAI,KAAK,GAAE,YAAY,EAAE,MAAM,GAAG,EAAE,CAAC;AACtD,QAAM,mBAAmB,yBAAyB,OAAO,QAAQ,gBAAgB,QAAQ;AACzF,QAAM,SAAS,mBAAmB,YAAY,OAAO,mBAAmB;AACxE,QAAM,UAAU,KAAK,YAAY,aAAa,YAAY,GAAG,MAAM;AAGnE,QAAM,YAAuB;AAAA,IAC3B,WAAU,oBAAI,KAAK,GAAE,YAAY;AAAA,IACjC,YAAY;AAAA,IACZ,SAAS,SAAS,WAAW,CAAC;AAAA,IAC9B,aAAa,SAAS,eAAe;AAAA,IACrC,QAAQ,OAAO;AAAA,MACb,gBAAgB,IAAI,CAAC,MAAM,CAAC,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;AAAA,IAC9E;AAAA,EACF;AACA,QAAM,UAAU,KAAK,YAAY,aAAa,qBAAqB,GAAG,KAAK,UAAU,WAAW,MAAM,CAAC,CAAC;AAExG,QAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AAEvD,SAAO;AAAA,IACL,aAAa,gBAAgB;AAAA,IAC7B,WAAW,KAAK,UAAU;AAAA,IAC1B,eAAe,KAAK,cAAc;AAAA,IAClC,iBAAiB,KAAK,gBAAgB;AAAA,IACtC,eAAe,KAAK,cAAc;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;;;AClbA,SAAS,YAAAA,iBAAgB;AACzB,SAAS,QAAAC,aAAY;AAarB,eAAsB,iBACpB,YACA,SACe;AAEf,MAAI,qBAAqB,QAAQ;AACjC,MAAI,kBAAkB,QAAQ;AAE9B,MAAI,QAAQ,SAAS;AACnB,QAAI;AACF,YAAM,MAAM,MAAMD,UAASC,MAAK,YAAY,aAAa,qBAAqB,GAAG,OAAO;AACxF,YAAM,YAAuB,KAAK,MAAM,GAAG;AAC3C,2BAAqB,sBAAsB,UAAU;AACrD,UAAI,CAAC,mBAAmB,UAAU,QAAQ,SAAS,GAAG;AACpD,0BAAkB,UAAU,QAAQ,KAAK,GAAG;AAAA,MAC9C;AACA,UAAI,CAAC,QAAQ,MAAM;AACjB,QAAO,KAAK,mBAAmB,UAAU,UAAU,KAAK,UAAU,QAAQ,KAAK,IAAI,KAAK,aAAa,GAAG;AAAA,MAC1G;AAAA,IACF,QAAQ;AACN,MAAO,MAAM,2EAA2E;AACxF,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AAGA,QAAM,aAAa,MAAM,kBAAkB,YAAY,kBAAkB;AACzE,QAAM,YAAY,gBAAgB,YAAY,UAAU;AAExD,QAAM,UAAU,kBACZ,gBAAgB,MAAM,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,IAC9C;AAEJ,QAAM,OAAO,QAAQ,OAAO,OAAc,QAAQ,iBAAiB,WAAW,IAAI,MAAM;AAExF,MAAI;AACF,UAAM,UAAU,QAAQ;AAExB,QAAI,MAAM;AACR,WAAK,OAAO;AAAA,IACd;AAEA,UAAM,SAAS,MAAM;AAAA,MACnB;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,WAAW;AAAA,MACX;AAAA,QACE;AAAA,QACA,UAAU,QAAQ;AAAA,QAClB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,QACd,YAAY,OACR,CAAC,OAAO,OAAO,UAAU;AACvB,eAAK,OAAO,sBAAsB,KAAK,IAAI,KAAK,KAAK,KAAK;AAAA,QAC5D,IACA;AAAA,MACN;AAAA,IACF;AAEA,QAAI,MAAM;AACR,WAAK,KAAK;AAAA,IACZ;AAEA,QAAI,QAAQ,MAAM;AAChB,cAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,IAC7C,OAAO;AACL,YAAM,QAAQ;AAAA,QACZ,GAAG,OAAO,SAAS;AAAA,QACnB,GAAG,OAAO,aAAa;AAAA,QACvB,GAAG,OAAO,eAAe;AAAA,MAC3B;AACA,UAAI,OAAO,gBAAgB,GAAG;AAC5B,cAAM,KAAK,GAAG,OAAO,aAAa,UAAU;AAAA,MAC9C;AACA,MAAO;AAAA,QACL,UAAU,OAAO,WAAW,YAAY,MAAM,KAAK,IAAI,CAAC,SAAS,OAAO,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,MACvG;AACA,MAAO,OAAO,mCAAmC;AAEjD,UAAI,OAAO,OAAO,SAAS,GAAG;AAC5B,QAAO,QAAQ,GAAG,OAAO,OAAO,MAAM,gCAAgC;AACtE,mBAAW,OAAO,OAAO,OAAO,MAAM,GAAG,CAAC,GAAG;AAC3C,UAAO,OAAO,GAAG;AAAA,QACnB;AACA,YAAI,OAAO,OAAO,SAAS,GAAG;AAC5B,UAAO,OAAO,WAAW,OAAO,OAAO,SAAS,CAAC,OAAO;AAAA,QAC1D;AAAA,MACF;AAAA,IACF;AAAA,EACF,UAAE;AACA,UAAM,UAAU,WAAW;AAAA,EAC7B;AACF;","names":["readFile","join"]}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getTablesQuery
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-EHM6AMMA.js";
|
|
4
4
|
import {
|
|
5
5
|
createConnector,
|
|
6
6
|
resolveConnection
|
|
@@ -36,4 +36,4 @@ async function listTables(projectDir, options) {
|
|
|
36
36
|
export {
|
|
37
37
|
listTables
|
|
38
38
|
};
|
|
39
|
-
//# sourceMappingURL=tables-
|
|
39
|
+
//# sourceMappingURL=tables-5BVEKUPB.js.map
|
|
@@ -15,6 +15,8 @@ yamchart test # Run model assertions
|
|
|
15
15
|
yamchart tables # List database tables
|
|
16
16
|
yamchart describe <table> # Show table columns
|
|
17
17
|
yamchart search <keyword> # Find tables/columns
|
|
18
|
+
yamchart sync-warehouse # Sync warehouse metadata to catalog
|
|
19
|
+
yamchart lineage <model> # Show model dependency tree
|
|
18
20
|
yamchart advisor # AI-powered dbt model advisor
|
|
19
21
|
```
|
|
20
22
|
|