dbt-extra 0.1.0__py3-none-any.whl

dbt_extra/__init__.py

@@ -0,0 +1,18 @@
+"""Offline dbt transformation review, inspection, and checked source edits."""
+
+from dbt_extra.errors import SpecError
+
+__version__ = "0.1.0"
+
+__all__ = ["Project", "ChangeSet", "SpecError", "__version__"]
+
+
+def __getattr__(name):
+    # Version/error access (including CLI --version) needs no project loading.
+    if name == "Project":
+        from dbt_extra.project import Project
+        return Project
+    if name == "ChangeSet":
+        from dbt_extra.files.changes import ChangeSet
+        return ChangeSet
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

dbt_extra/__main__.py

@@ -0,0 +1,3 @@
+from dbt_extra.cli import main
+
+raise SystemExit(main())

dbt_extra/analysis/__init__.py

@@ -0,0 +1,5 @@
+"""Shared dbt analysis used by inspection, review, and enrichment."""
+
+from dbt_extra.analysis.analyzer import analyze, catalog
+
+__all__ = ["analyze", "catalog"]

dbt_extra/analysis/analyzer.py

@@ -0,0 +1,111 @@
+"""Shared best-effort dbt source analysis with explicit coverage and provenance."""
+
+from __future__ import annotations
+
+from sqlglot import exp
+from sqlglot.errors import SqlglotError
+from dbt_extra.analysis.lineage import column_lineage, qualify_columns
+from dbt_extra.analysis.models import ResourceAnalysis, coverage
+from dbt_extra.analysis.types import inferred_types
+from dbt_extra.sql.macros import macro_calls, parse_model, contains_opaque
+from dbt_extra.sql.references import resolve_ref
+from dbt_extra.errors import SpecError
+from dbt_extra.sql.scopes import relation_tables
+from dbt_extra.sql.renderer import sql, render
+
+
+def analyze(project, resource, *, expand=False) -> ResourceAnalysis:
+    result = {"resource_id": resource.id, "name": resource.name, "resource_type": resource.kind,
+              "package": resource.package, "model_path": resource.model_path, "source_file": resource.source_file,
+              "properties_file": resource.property_path.relative_to(project.root).as_posix() if resource.property_path else None,
+              "description": resource.properties.get("description", ""), "properties": resource.properties,
+              "config": resource.config, "sql": resource.code, "columns": [], "dependencies": [],
+              "status": "metadata_only", "reason": None, "provenance": "authored" if resource.path else "artifact"}
+    declared = resource.columns
+    outputs = {name: {"name": name, "expression": "", "description": props.get("description", ""),
+                      "data_type": props.get("data_type", ""), "properties": props,
+                      "upstream_columns": [], "status": "declared", "provenance": "authored" if resource.property_path else "artifact"}
+               for name, props in declared.items()}
+    if resource.kind == "model" and (not resource.path or resource.path.suffix == ".sql"):
+        try:
+            dialect = project.require_dialect()
+            parsed = parse_model(project, resource.code, resource.id, expand=expand, partial=True)
+            provenance = "expanded_macro_sql" if expand and macro_calls(resource.code) else "authored_sql"
+            result["config"] = {**result["config"], **parsed.config}
+            result["status"] = "parsed"
+            if parsed.opaque_macros:
+                result["status"], result["reason"] = "partial", "; ".join(dict.fromkeys(parsed.opaque_macros.values()))
+            dependencies = {}
+            schema = {}
+            for token, ref in parsed.refs.items():
+                upstream = resolve_ref(project, ref)
+                dependencies[token] = upstream.id if upstream else ref.text
+                if upstream:
+                    if upstream.known_columns:
+                        schema[token] = {name: p.get("data_type") or "UNKNOWN" for name, p in upstream.known_columns.items()}
+            result["dependencies"] = list(dict.fromkeys(dependencies.values()))
+            result["context"] = {"query": resource.code, "relations": [sql(t, dialect) for t in relation_tables(parsed.tree)],
+                                 "note": "Expressions depend on complete query context, including joins, filters, grouping and multiplicity"}
+            projections = parsed.tree.selects
+            counts = {}
+            for p in projections:
+                counts[p.output_name] = counts.get(p.output_name, 0) + 1
+            for projection in projections:
+                if contains_opaque(projection, parsed):
+                    continue  # A template boundary may emit any number of columns.
+                name = projection.output_name
+                if not name or projection.is_star or counts[name] != 1:
+                    result["status"], result["reason"] = "partial", "Wildcard, unnamed, or duplicate output columns require additional schema evidence"
+                    continue
+                item = outputs.setdefault(name, {"name": name, "description": "", "data_type": "", "properties": {}, "upstream_columns": []})
+                expression = projection.this if isinstance(projection, exp.Alias) else projection
+                expression_text = render(expression, parsed, dialect, pretty=False).strip().rstrip(";")
+                # Template relations in scalar subqueries remain readable; expression
+                # writeback itself is restricted to independently parseable expressions.
+                item.update(expression=expression_text, status="parsed", provenance=provenance)
+            # Unknown shapes cannot supply schema/type/lineage evidence. Keep
+            # independently parsed expressions, without qualifying through them.
+            if not parsed.opaque_macros:
+                try:
+                    qualified = qualify_columns(parsed.tree, schema, dialect)
+                    for projection in qualified.selects:
+                        name = projection.output_name
+                        if name and not projection.is_star and name not in outputs:
+                            expression = projection.this if isinstance(projection, exp.Alias) else projection
+                            outputs[name] = {"name": name, "description": "", "data_type": "", "properties": {},
+                                "upstream_columns": [], "expression": render(expression, parsed, dialect, pretty=False).strip().rstrip(";"), "status": "parsed", "provenance": provenance}
+                    if not any(p.is_star for p in qualified.selects) and result["reason"] == "Wildcard, unnamed, or duplicate output columns require additional schema evidence":
+                        result["status"], result["reason"] = "parsed", None
+                    types = inferred_types(qualified, schema, dialect)
+                    lineage_for = column_lineage(qualified, schema, dialect, dependencies)
+                    output_names = {p.output_name for p in qualified.selects if not p.is_star}
+                    for name, item in outputs.items():
+                        if name not in output_names:
+                            continue
+                        if name in types:
+                            item["inferred_type"] = types[name]
+                        leaves = lineage_for(name)
+                        item["upstream_columns"] = leaves
+                        item["status"] = "resolved" if leaves else "parsed"
+                except (SqlglotError, ValueError, KeyError) as exc:
+                    result["status"], result["reason"] = "partial", f"Column qualification unavailable: {exc}"
+        except (SpecError, SqlglotError) as exc:
+            result["status"], result["reason"] = "unavailable", str(exc)
+    elif resource.kind == "macro":
+        result["status"], result["reason"] = "source_only", "Macro files are editable source, not expanded SQL"
+    elif resource.kind == "snapshot":
+        result["reason"] = "Snapshots retain their execution/history boundary"
+    if not resource.path:
+        result["reason"] = "Artifact/package resource is read-only; source freshness is unverified"
+    result["columns"] = list(outputs.values())
+    return result
+
+
+def catalog(project, resources, *, expand=False):
+    models = [analyze(project, resource, expand=expand) for resource in resources]
+    return {"format_version": 1, "project": project.name, "dialect": project.dialect,
+            "source_of_truth": "dbt", "resources": models, "diagnostics": project.diagnostics,
+            "artifact_evidence": {"manifest": str(project.manifest_path) if project.manifest else None,
+                                  "generated_at": project.manifest.get("metadata", {}).get("generated_at"),
+                                  "compiled_sql_used_for_rewriting": False},
+            "coverage": coverage(models)}

dbt_extra/analysis/dependencies.py

@@ -0,0 +1,26 @@
+"""Resource dependency graphs and cycle checks."""
+
+from dbt_extra.errors import SpecError
+from dbt_extra.sql.references import resolve_ref
+
+
+def dependencies(project, parsed):
+    return [up.id for ref in parsed.refs.values() if (up := resolve_ref(project, ref))]
+
+
+def validate_cycles(graph):
+    active, done = [], set()
+
+    def visit(uid):
+        if uid in active:
+            raise SpecError(f"Cyclic dbt references: {' -> '.join((*active, uid))}")
+        if uid in done:
+            return
+        active.append(uid)
+        for dependency in graph.get(uid, []):
+            visit(dependency)
+        active.pop()
+        done.add(uid)
+
+    for uid in graph:
+        visit(uid)

dbt_extra/analysis/lineage.py

@@ -0,0 +1,35 @@
+"""Conservative qualification and direct column lineage."""
+
+from sqlglot import exp
+from sqlglot.lineage import lineage
+from sqlglot.optimizer.qualify import qualify
+from sqlglot.optimizer.scope import build_scope
+
+from dbt_extra.errors import SpecError
+from dbt_extra.sql.qualification import bind_single_inputs, check_bindings
+from dbt_extra.sql.renderer import sql
+
+
+def qualify_columns(tree, schema, dialect):
+    try:
+        schema = check_bindings(tree, schema, dialect, strict=False)
+    except SpecError as exc:
+        raise ValueError(str(exc)) from exc
+    return qualify(bind_single_inputs(tree.copy(), schema, dialect), dialect=dialect,
+                   schema=schema, validate_qualify_columns=True)
+
+
+def column_lineage(tree, schema, dialect, dependencies):
+    scope = build_scope(tree)
+
+    def for_column(name):
+        graph = lineage(name, tree, schema=schema, dialect=dialect, scope=scope, trim_selects=False)
+        leaves = []
+        for node in graph.walk():
+            if not node.downstream and isinstance(node.expression, exp.Table):
+                relation = dependencies.get(node.expression.name, sql(node.expression, dialect))
+                column = node.name.rsplit(".", 1)[-1].strip('"`[]')
+                leaves.append({"resource_id": relation, "column": column})
+        return leaves
+
+    return for_column

dbt_extra/analysis/models.py

@@ -0,0 +1,34 @@
+"""Coverage and provenance vocabulary for JSON-compatible analysis results.
+
+Results describe dbt definitions, not an independently authored semantic model.
+"""
+
+from collections import Counter
+from typing import Literal, NotRequired, TypedDict
+
+
+Status = Literal["metadata_only", "parsed", "partial", "unavailable", "source_only"]
+
+
+class ResourceAnalysis(TypedDict):
+    resource_id: str
+    name: str
+    resource_type: str
+    package: str
+    model_path: str
+    source_file: str
+    properties_file: str | None
+    description: str
+    properties: dict
+    config: dict
+    sql: str
+    columns: list[dict]
+    dependencies: list[str]
+    status: Status
+    reason: str | None
+    provenance: str
+    context: NotRequired[dict]
+
+
+def coverage(resources):
+    return dict(sorted(Counter(r["status"] for r in resources).items()))

dbt_extra/analysis/types.py

@@ -0,0 +1,12 @@
+"""Best-effort type inference from qualified SQL and declared schema evidence."""
+
+from sqlglot import exp
+from sqlglot.optimizer.annotate_types import annotate_types
+
+from dbt_extra.sql.renderer import sql
+
+
+def inferred_types(tree, schema, dialect):
+    typed = annotate_types(tree, schema=schema, dialect=dialect)
+    return {p.output_name: sql(p.type, dialect) for p in typed.selects
+            if not p.is_star and p.type and p.type.this not in {exp.DataType.Type.UNKNOWN, exp.DataType.Type.NULL}}

dbt_extra/cli.py

@@ -0,0 +1,243 @@
+"""dbt-native command orchestration; no dbt or warehouse execution."""
+
+from functools import wraps
+from pathlib import Path
+from zipfile import BadZipFile
+
+import click
+from sqlglot.errors import SqlglotError
+
+from dbt_extra import __version__
+from dbt_extra.enrichment import enrich
+from dbt_extra.files.changes import ChangeSet
+from dbt_extra.project import Project
+from dbt_extra.review import export_review, import_review
+from dbt_extra.inspection import inspect, plan
+from dbt_extra.optimization import optimize
+from dbt_extra.sql.renderer import render
+from dbt_extra.validation import validate as validate_project
+from dbt_extra.errors import SpecError
+from dbt_extra.files.yaml import dump_yaml
+
+
+class Names(click.Option):
+    """A repeatable option accepting one or more space-separated selectors."""
+    def __init__(self, *args, **kwargs):
+        kwargs["type"] = click.UNPROCESSED
+        super().__init__(*args, **kwargs)
+
+    def add_to_parser(self, parser, ctx):
+        super().add_to_parser(parser, ctx)
+        option = next((parser._long_opt.get(name) or parser._short_opt.get(name) for name in self.opts
+                       if parser._long_opt.get(name) or parser._short_opt.get(name)), None)
+        original = option.process
+        def process(value, state):
+            values = [value]
+            while state.rargs and not state.rargs[0].startswith("-"):
+                values.append(state.rargs.pop(0))
+            original(tuple(values), state)
+        option.process = process
+
+
+def names(values):
+    return [name for group in values for name in group] if values else None
+
+
+def guarded(function):
+    @wraps(function)
+    def run(*args, **kwargs):
+        try:
+            return function(*args, **kwargs)
+        except (ValueError, OSError, UnicodeError, BadZipFile, SqlglotError) as exc:
+            raise click.ClickException(str(exc)) from exc
+    return run
+
+
+def loaded(ctx, dialect=None):
+    return Project(ctx.obj["directory"], dialect=dialect or ctx.obj["dialect"], manifest=ctx.obj["manifest"])
+
+
+def selectors(function):
+    function = click.option("--exclude", cls=Names, multiple=True, help="Selectors to exclude; explicit lists replace configuration.")(function)
+    return click.option("--select", "-s", cls=Names, multiple=True, help="Names, resource IDs, or path:/tag:/package:/resource_type: selectors.")(function)
+
+
+def preview_option(function):
+    return click.option("--preview", "--dry-run", is_flag=True, help="Show proposed changes without writing files.")(function)
+
+
+def summary(changes, preview):
+    for path in changes:
+        click.echo(f"update: {path}")
+    click.echo(f"{'Preview: ' if preview else ''}{len(changes)} file change(s)")
+
+
+def finish(changes, preview):
+    if preview:
+        click.echo(changes.diffs(), nl=False)
+    summary(changes.apply(preview=preview), preview)
+
+
+@click.group()
+@click.version_option(version=__version__, prog_name="extra")
+@click.option("--project-dir", type=click.Path(path_type=Path), envvar="EXTRA_PROJECT_DIR")
+@click.option("--dialect", help="SQL dialect; overrides configuration and manifest adapter.")
+@click.option("--manifest", type=click.Path(exists=True, dir_okay=False, path_type=Path), help="Optional dbt manifest evidence.")
+@click.pass_context
+def cli(ctx, project_dir, dialect, manifest):
+    """Review, inspect, and edit native dbt transformations offline."""
+    ctx.obj = {"directory": project_dir, "dialect": dialect, "manifest": manifest}
+
+
+@cli.command("init")
+@click.argument("directory", required=False, default=".", type=click.Path(path_type=Path))
+@preview_option
+@click.pass_context
+@guarded
+def init_command(ctx, directory, preview):
+    """Add optional dbt-extra settings to an existing native dbt project."""
+    base = ctx.obj["directory"] or Path.cwd()
+    root = (base / directory).resolve()
+    if not (root / "dbt_project.yml").is_file():
+        raise SpecError("init requires an existing dbt_project.yml; dbt-extra does not scaffold a second transformation source")
+    path = root / "extra_project.yaml"
+    if path.exists():
+        raise SpecError(f"Refusing to overwrite {path}")
+    changes = ChangeSet(root)
+    changes.set(path, dump_yaml({"version": 2, "analysis": {"dialect": ctx.obj["dialect"] or "auto"},
+        "export": {"output_dir": "extra/review", "format": "xlsx"}, "optimization": {"format": "pretty", "preserve_macros": True}}))
+    finish(changes, preview)
+
+
+@cli.command()
+@click.option("--dialect")
+@click.pass_context
+@guarded
+def validate(ctx, dialect):
+    """Check local structure, static SQL, and reference cycles; never run dbt."""
+    project = loaded(ctx, dialect)
+    skipped = validate_project(project)
+    for uid, reason in skipped.items():
+        click.echo(f"Unavailable {uid}: {reason}", err=True)
+    click.echo(f"Validated {len(project.resources)} dbt resources offline; {len(skipped)} SQL analysis unavailable. Not a dbt parse/build.")
+
+
+@cli.command("inspect")
+@click.argument("table")
+@click.option("--columns", cls=Names, multiple=True)
+@click.option("--from", "from_tables", cls=Names, multiple=True, help="Upstream models to retain as relation boundaries.")
+@click.option("--optimize", "optimized", type=click.BOOL, default=None)
+@click.option("--expand", type=click.BOOL, default=None, help="Expand upstream models and statically resolvable local macros.")
+@click.option("--compile", "compiled", type=click.BOOL, default=None, help="Resolve physical relation names offline (default: true); does not run dbt compile.")
+@click.option("--view", type=click.Choice(["sql", "plan"]), default="sql")
+@click.option("--dialect")
+@click.pass_context
+@guarded
+def inspect_command(ctx, table, columns, from_tables, optimized, expand, compiled, view, dialect):
+    """Explain full model or selected-column logic without changing files."""
+    project = loaded(ctx, dialect)
+    settings = project.options("inspection", {"optimize": optimized, "expand": expand, "compile": compiled, "boundaries": names(from_tables)})
+    tree, parsed, issues = inspect(project, table, names(columns), boundaries=settings.get("boundaries", []),
+        expanded=settings.get("expand", True), optimized=settings.get("optimize", True), compiled=settings.get("compile", True))
+    click.echo("-- Definition explanation, not stored-relation equivalence or a warehouse execution plan.")
+    click.echo("-- Offline relation resolution; unresolved dbt templates retained. Not executed. Authored files unchanged."
+               if settings.get("compile", True) else "-- Authored dbt ref/source boundaries retained; not executed. Authored files unchanged.")
+    for issue in issues:
+        click.echo(f"-- Boundary/coverage: {issue['resource_id']}: {issue['reason']}".replace("\n", " "))
+    click.echo(plan(tree, parsed, project.dialect) if view == "plan" else render(tree, parsed, project.dialect), nl=view == "plan")
+
+
+@cli.command("export")
+@click.option("--format", type=click.Choice(["xlsx", "csv", "json"]))
+@click.option("--output-dir", "--path", type=click.Path(path_type=Path))
+@click.option("--view", help="Named view from export.views.")
+@click.option("--expand/--no-expand", default=None, help="Show read-only SQL/columns from statically resolvable local macros.")
+@selectors
+@preview_option
+@click.option("--dialect")
+@click.pass_context
+@guarded
+def export_command(ctx, format, output_dir, view, expand, select, exclude, preview, dialect):
+    """Export tabular XLSX/CSV review or a read-only JSON analysis catalog."""
+    project = loaded(ctx, dialect)
+    options = project.options("export", {"format": format, "output_dir": output_dir, "select": names(select), "exclude": names(exclude), "expand": expand}, view=view)
+    paths, coverage = export_review(project, format=options.get("format", "xlsx"), output_dir=options.get("output_dir"),
+        select=options.get("select"), exclude=options.get("exclude"), preview=preview, expand=options.get("expand", False))
+    summary(paths, preview)
+    click.echo("Coverage: " + ", ".join(f"{k}={v}" for k, v in coverage.items()))
+
+
+@cli.command("import")
+@click.argument("path", type=click.Path(exists=True, path_type=Path))
+@preview_option
+@click.option("--dialect")
+@click.pass_context
+@guarded
+def import_command(ctx, path, preview, dialect):
+    """Apply supported XLSX/CSV edits to their original native dbt files."""
+    project = loaded(ctx, dialect)
+    changes = import_review(project, path)
+    for uid, reason in changes.metadata_skipped.items():
+        click.echo(f"Skipped metadata {uid}: {reason}", err=True)
+    finish(changes, preview)
+
+
+@cli.command("optimize")
+@selectors
+@click.option("--format", type=click.Choice(["pretty", "compact"]))
+@click.option("--preserve-macro", "preserve_macros", type=click.BOOL, default=None)
+@click.option("--enrich", "enrich_enabled", type=click.BOOL, default=None)
+@preview_option
+@click.option("--dialect")
+@click.pass_context
+@guarded
+def optimize_command(ctx, select, exclude, format, preserve_macros, enrich_enabled, preview, dialect):
+    """Conservatively rewrite authored dbt SQL, never compiled SQL."""
+    project = loaded(ctx, dialect)
+    options = project.options("optimization", {"select": names(select), "exclude": names(exclude), "format": format,
+        "preserve_macros": preserve_macros, "enrich": enrich_enabled})
+    changes, skipped = optimize(project, select=options.get("select"), exclude=options.get("exclude"),
+        format=options.get("format", "pretty"), preserve_macros=options.get("preserve_macros", True))
+    for uid, reason in skipped.items():
+        click.echo(f"Skipped {uid}: {reason}. Model unchanged.", err=True)
+    optimization_report = changes.optimization_report
+    counts = {status: sum(r["status"] == status for r in optimization_report.values())
+              for status in ("rewritten", "qualified", "formatted", "unchanged")}
+    click.echo(f"Optimization: {len(optimization_report)} accepted "
+               f"({counts['rewritten']} structurally rewritten, {counts['qualified']} qualified, "
+               f"{counts['formatted']} formatting-only, {counts['unchanged']} unchanged); "
+                f"{sum(r['changed'] for r in optimization_report.values())} proposed SQL change(s); {len(skipped)} skipped")
+    for uid, report in optimization_report.items():
+        for reason in report["retained"]:
+            click.echo(f"Retained {uid}: {reason}")
+    if options.get("enrich", False):
+        changes, report = enrich(project, select=options.get("select"), exclude=options.get("exclude"), changes=changes)
+        enrichment_summary(report)
+    finish(changes, preview)
+    click.echo(f"Optimization SQL files written: {0 if preview else sum(r['changed'] for r in optimization_report.values())}")
+
+
+def enrichment_summary(report):
+    click.echo(f"Metadata: {sum('value' in r for r in report)} missing properties inferred; {sum('reason' in r for r in report)} unresolved")
+    for item in report:
+        if "reason" in item and "column" not in item:
+            click.echo(f"Skipped metadata {item['resource_id']}: {item['reason']}", err=True)
+
+
+@cli.command("enrich")
+@selectors
+@preview_option
+@click.option("--dialect")
+@click.pass_context
+@guarded
+def enrich_command(ctx, select, exclude, preview, dialect):
+    """Fill missing inferred types and safe passthrough descriptions in dbt YAML."""
+    project = loaded(ctx, dialect)
+    options = project.options("metadata", {"select": names(select), "exclude": names(exclude)})
+    changes, report = enrich(project, select=options.get("select"), exclude=options.get("exclude"))
+    enrichment_summary(report)
+    finish(changes, preview)
+
+
+def main(argv=None):
+    return cli.main(args=argv, prog_name="extra")

dbt_extra/config.py

@@ -0,0 +1,71 @@
+"""Optional tool settings, project resolution, and explicit option overrides."""
+
+import os
+from pathlib import Path
+
+from dbt_extra.errors import SpecError
+from dbt_extra.files.yaml import fields, mapping
+
+
+def resolve_project(project_dir=None):
+    value = project_dir if project_dir is not None else os.environ.get("EXTRA_PROJECT_DIR")
+    if value is not None:
+        return Path(value).absolute() / "extra_project.yaml"
+    for directory in (Path.cwd(), *Path.cwd().parents):
+        candidate = directory / "extra_project.yaml"
+        if candidate.is_file():
+            return candidate
+    return Path.cwd() / "extra_project.yaml"
+
+
+def strings(value, context):
+    if not isinstance(value, list) or any(not isinstance(v, str) or not v for v in value):
+        raise SpecError(f"{context}: expected a list of nonempty strings")
+    return value
+
+
+def validate_settings(settings):
+    allowed = {
+        "dbt": {"project_dir", "artifacts"}, "analysis": {"dialect"},
+        "inspection": {"expand", "optimize", "compile", "boundaries"},
+        "export": {"output_dir", "default_view", "format", "select", "exclude", "views", "expand"},
+        "optimization": {"format", "select", "exclude", "preserve_macros", "enrich"},
+        "metadata": {"select", "exclude"},
+    }
+    fields(settings, {"version", *allowed}, set(), "extra_project.yaml")
+    if settings.get("version", 2) != 2:
+        raise SpecError("extra_project.yaml must use version: 2 tool configuration; models are dbt files")
+    for key, keys in allowed.items():
+        fields(mapping(settings.get(key, {}), key), keys, set(), key)
+    for section, options in settings.items():
+        if not isinstance(options, dict):
+            continue
+        for key, value in options.items():
+            if key in {"select", "exclude", "boundaries"}:
+                strings(value, f"{section}.{key}")
+            if key in {"expand", "optimize", "compile", "preserve_macros", "enrich"} and type(value) is not bool:
+                raise SpecError(f"{section}.{key}: expected a boolean")
+            if key in {"output_dir", "default_view", "format", "dialect", "project_dir"} and (not isinstance(value, str) or not value):
+                raise SpecError(f"{section}.{key}: expected nonempty text")
+    views = mapping(settings.get("export", {}).get("views", {}), "export.views")
+    for name, view in views.items():
+        fields(mapping(view, f"export.views.{name}"), {"format", "select", "exclude", "expand"}, set(), f"export.views.{name}")
+        if "expand" in view and type(view["expand"]) is not bool:
+            raise SpecError(f"export.views.{name}.expand: expected a boolean")
+        for key in {"select", "exclude"} & view.keys():
+            strings(view[key], f"export.views.{name}.{key}")
+
+
+def options(settings, section, overrides=None, *, view=None):
+    result = dict(settings.get(section, {}))
+    views = result.pop("views", {})
+    if section == "export":
+        selected = view or result.get("default_view")
+        if selected:
+            if selected not in views:
+                raise SpecError(f"Unknown export view: {selected}")
+            setting = mapping(views[selected], f"export.views.{selected}")
+            fields(setting, {"format", "select", "exclude", "expand"}, set(), f"export.views.{selected}")
+            result.update(setting)
+    result.update({k: v for k, v in (overrides or {}).items() if v is not None})
+    return result

dbt_extra/enrichment.py

@@ -0,0 +1,84 @@
+"""Fill missing metadata using shared analysis; never overwrite authored values."""
+
+from __future__ import annotations
+
+import copy
+
+from sqlglot import exp
+
+from dbt_extra.analysis import analyze
+from dbt_extra.files.changes import ChangeSet, digest
+from dbt_extra.sql.parser import parse_authored
+from dbt_extra.errors import SpecError
+
+
+def enrich(project, *, select=None, exclude=None, changes=None):
+    project.require_dialect()
+    changes = changes or ChangeSet(project.root)
+    for path, data in project.contents.items():
+        changes.watch(path, digest(data), check=True, external=not path.is_relative_to(project.root))
+    report, cache, active = [], {}, set()
+
+    def metadata(r):
+        if r.id in cache:
+            return cache[r.id]
+        if r.id in active:
+            return r.columns
+        active.add(r.id)
+        result = copy.deepcopy(r.columns)
+        analysis = analyze(project, r)
+        try:
+            parsed = parse_authored(r.code, project.dialect, r.id)
+            output = {p.output_name: p for p in parsed.tree.selects}
+        except SpecError:
+            parsed, output = None, {}
+        for item in analysis["columns"]:
+            name = item["name"]
+            props = result.setdefault(name, {"name": name})
+            if not props.get("data_type") and item.get("inferred_type"):
+                props["data_type"] = item["inferred_type"]
+            projection = output.get(name)
+            expression = projection.this if isinstance(projection, exp.Alias) else projection
+            # Only bare passthrough/renamed columns inherit a description. The
+            # lineage relation must resolve uniquely; transformed values do not.
+            if not props.get("description") and isinstance(expression, exp.Column):
+                leaves = item["upstream_columns"]
+                if len(leaves) == 1:
+                    leaf = leaves[0]
+                    upstream = project.resources.get(leaf["resource_id"])
+                    if upstream:
+                        inherited = metadata(upstream).get(leaf["column"], {}).get("description")
+                        if inherited:
+                            props["description"] = inherited
+        active.remove(r.id)
+        cache[r.id] = result
+        return result
+
+    for r in project.select(select, exclude, kinds={"model"}):
+        if not r.path or r.path.suffix != ".sql" or r.properties.get("versions"):
+            report.append({"resource_id": r.id, "reason": "Read-only or versioned metadata"})
+            continue
+        if not r.property_path:
+            report.append({"resource_id": r.id, "reason": "No existing writable dbt YAML entry; add one in dbt and retry"})
+            continue
+        proposed = copy.deepcopy(r.properties)
+        columns = proposed.setdefault("columns", [])
+        by_name = {c["name"]: c for c in columns}
+        for name, inferred in metadata(r).items():
+            target = by_name.get(name)
+            for key in ("data_type", "description"):
+                if target and target.get(key):
+                    continue
+                if inferred.get(key):
+                    if target is None:
+                        target = {"name": name}
+                        columns.append(target)
+                        by_name[name] = target
+                    target[key] = inferred[key]
+                    report.append({"resource_id": r.id, "column": name, "property": key, "value": inferred[key]})
+                else:
+                    report.append({"resource_id": r.id, "column": name, "property": key, "reason": "Not safely inferable from available source metadata"})
+        proposed.setdefault("name", r.name)
+        if any("value" in item and item["resource_id"] == r.id for item in report):
+            changes.properties(project, r, proposed)
+    return changes, report

dbt_extra/errors.py

@@ -0,0 +1,5 @@
+"""Errors shared by project loading, analysis, and checked source edits."""
+
+
+class SpecError(ValueError):
+    """Invalid project data, unsupported analysis, or an unsafe file operation."""