sqlmesh 0.209.1.dev12__py3-none-any.whl → 0.209.1.dev13__py3-none-any.whl

sqlmesh/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.209.1.dev12'
-__version_tuple__ = version_tuple = (0, 209, 1, 'dev12')
+__version__ = version = '0.209.1.dev13'
+__version_tuple__ = version_tuple = (0, 209, 1, 'dev13')
 
 __commit_id__ = commit_id = None

sqlmesh/core/audit/definition.py

@@ -438,13 +438,15 @@ def load_audit(
 
     extra_kwargs: t.Dict[str, t.Any] = {}
     if is_standalone:
-        jinja_macro_refrences, used_variables = extract_macro_references_and_variables(
+        jinja_macro_refrences, referenced_variables = extract_macro_references_and_variables(
             *(gen(s) for s in statements),
             gen(query),
         )
         jinja_macros = (jinja_macros or JinjaMacroRegistry()).trim(jinja_macro_refrences)
         for jinja_macro in jinja_macros.root_macros.values():
-            used_variables.update(extract_macro_references_and_variables(jinja_macro.definition)[1])
+            referenced_variables.update(
+                extract_macro_references_and_variables(jinja_macro.definition)[1]
+            )
 
         extra_kwargs["jinja_macros"] = jinja_macros
         extra_kwargs["python_env"] = make_python_env(
@@ -453,7 +455,7 @@ def load_audit(
             module_path,
             macros or macro.get_registry(),
             variables=variables,
-            used_variables=used_variables,
+            referenced_variables=referenced_variables,
         )
         extra_kwargs["default_catalog"] = default_catalog
         if project is not None:

sqlmesh/core/constants.py

@@ -80,7 +80,9 @@ REQUIREMENTS = "sqlmesh-requirements.lock"
 DEFAULT_SCHEMA = "default"
 
 SQLMESH_VARS = "__sqlmesh__vars__"
+SQLMESH_VARS_METADATA = "__sqlmesh__vars__metadata__"
 SQLMESH_BLUEPRINT_VARS = "__sqlmesh__blueprint__vars__"
+SQLMESH_BLUEPRINT_VARS_METADATA = "__sqlmesh__blueprint__vars__metadata__"
 
 VAR = "var"
 BLUEPRINT_VAR = "blueprint_var"

sqlmesh/core/macros.py

@@ -210,7 +210,12 @@ class MacroEvaluator:
                 self.macros[normalize_macro_name(k)] = self.env[k]
             elif v.is_value:
                 value = self.env[k]
-                if k in (c.SQLMESH_VARS, c.SQLMESH_BLUEPRINT_VARS):
+                if k in (
+                    c.SQLMESH_VARS,
+                    c.SQLMESH_VARS_METADATA,
+                    c.SQLMESH_BLUEPRINT_VARS,
+                    c.SQLMESH_BLUEPRINT_VARS_METADATA,
+                ):
                     value = {
                         var_name: (
                             self.parse_one(var_value.sql)
@@ -557,17 +562,25 @@ class MacroEvaluator:
 
     def var(self, var_name: str, default: t.Optional[t.Any] = None) -> t.Optional[t.Any]:
         """Returns the value of the specified variable, or the default value if it doesn't exist."""
-        return (self.locals.get(c.SQLMESH_VARS) or {}).get(var_name.lower(), default)
+        return {
+            **(self.locals.get(c.SQLMESH_VARS) or {}),
+            **(self.locals.get(c.SQLMESH_VARS_METADATA) or {}),
+        }.get(var_name.lower(), default)
 
     def blueprint_var(self, var_name: str, default: t.Optional[t.Any] = None) -> t.Optional[t.Any]:
         """Returns the value of the specified blueprint variable, or the default value if it doesn't exist."""
-        return (self.locals.get(c.SQLMESH_BLUEPRINT_VARS) or {}).get(var_name.lower(), default)
+        return {
+            **(self.locals.get(c.SQLMESH_BLUEPRINT_VARS) or {}),
+            **(self.locals.get(c.SQLMESH_BLUEPRINT_VARS_METADATA) or {}),
+        }.get(var_name.lower(), default)
 
     @property
     def variables(self) -> t.Dict[str, t.Any]:
         return {
             **self.locals.get(c.SQLMESH_VARS, {}),
+            **self.locals.get(c.SQLMESH_VARS_METADATA, {}),
             **self.locals.get(c.SQLMESH_BLUEPRINT_VARS, {}),
+            **self.locals.get(c.SQLMESH_BLUEPRINT_VARS_METADATA, {}),
         }
 
     def _coerce(self, expr: exp.Expression, typ: t.Any, strict: bool = False) -> t.Any:

sqlmesh/core/model/common.py

@@ -28,7 +28,7 @@ if t.TYPE_CHECKING:
     from sqlmesh.utils import registry_decorator
     from sqlmesh.utils.jinja import MacroReference
 
-    MacroCallable = registry_decorator
+    MacroCallable = t.Union[Executable, registry_decorator]
 
 
 def make_python_env(
@@ -40,7 +40,7 @@ def make_python_env(
     module_path: Path,
     macros: MacroRegistry,
     variables: t.Optional[t.Dict[str, t.Any]] = None,
-    used_variables: t.Optional[t.Set[str]] = None,
+    referenced_variables: t.Optional[t.Set[str]] = None,
     path: t.Optional[Path] = None,
     python_env: t.Optional[t.Dict[str, Executable]] = None,
     strict_resolution: bool = True,
@@ -48,20 +48,64 @@ def make_python_env(
     dialect: DialectType = None,
 ) -> t.Dict[str, Executable]:
     python_env = {} if python_env is None else python_env
-    variables = variables or {}
     env: t.Dict[str, t.Tuple[t.Any, t.Optional[bool]]] = {}
-    used_macros: t.Dict[
-        str,
-        t.Tuple[t.Union[Executable | MacroCallable], t.Optional[bool]],
-    ] = {}
-    used_variables = (used_variables or set()).copy()
+
+    variables = variables or {}
+    blueprint_variables = blueprint_variables or {}
+
+    used_macros: t.Dict[str, t.Tuple[MacroCallable, bool]] = {}
+
+    # var -> True: var is metadata-only
+    # var -> False: var is not metadata-only
+    # var -> None: cannot determine whether var is metadata-only yet, need to walk macros first
+    used_variables: t.Dict[str, t.Optional[bool]] = dict.fromkeys(
+        referenced_variables or set(), False
+    )
+
+    # id(expr) -> true: expr appears under the AST of a metadata-only macro function
+    # id(expr) -> false: expr appears under the AST of a macro function whose metadata status we don't yet know
+    expr_under_metadata_macro_func: t.Dict[int, bool] = {}
+
+    # For @m1(@m2(@x), @y), we'd get x -> m1 and y -> m1
+    outermost_macro_func_ancestor_by_var: t.Dict[str, str] = {}
+    visited_macro_funcs: t.Set[int] = set()
+
+    def _is_metadata_var(
+        name: str, expression: exp.Expression, appears_in_metadata_expression: bool
+    ) -> t.Optional[bool]:
+        is_metadata_so_far = used_variables.get(name, True)
+        if is_metadata_so_far is False:
+            # We've concluded this variable is definitely not metadata-only
+            return False
+
+        appears_under_metadata_macro_func = expr_under_metadata_macro_func.get(id(expression))
+        if is_metadata_so_far and (
+            appears_in_metadata_expression or appears_under_metadata_macro_func
+        ):
+            # The variable appears in a metadata expression, e.g., audits (...),
+            # or in the AST of metadata-only macro call, e.g., @FOO(@x)
+            return True
+
+        # The variable appears in the AST of a macro call, but we don't know if it's metadata-only
+        if appears_under_metadata_macro_func is False:
+            return None
+
+        # The variable appears elsewhere, e.g., in the model's query: SELECT @x
+        return False
+
+    def _is_metadata_macro(name: str, appears_in_metadata_expression: bool) -> bool:
+        if name in used_macros:
+            is_metadata_so_far = used_macros[name][1]
+            return is_metadata_so_far and appears_in_metadata_expression
+
+        return appears_in_metadata_expression
 
     expressions = ensure_list(expressions)
     for expression_metadata in expressions:
         if isinstance(expression_metadata, tuple):
             expression, is_metadata = expression_metadata
         else:
-            expression, is_metadata = expression_metadata, None
+            expression, is_metadata = expression_metadata, False
 
         if isinstance(expression, d.Jinja):
             continue
@@ -72,31 +116,51 @@ def make_python_env(
                 if name not in macros:
                     continue
 
-                # If this macro has been seen before as a non-metadata macro, prioritize that
-                used_macros[name] = (
-                    macros[name],
-                    used_macros.get(name, (None, is_metadata))[1] and is_metadata,
-                )
-                if name == c.VAR:
+                used_macros[name] = (macros[name], _is_metadata_macro(name, is_metadata))
+
+                if name in (c.VAR, c.BLUEPRINT_VAR):
                     args = macro_func_or_var.this.expressions
                     if len(args) < 1:
-                        raise_config_error("Macro VAR requires at least one argument", path)
+                        raise_config_error(
+                            f"Macro {name.upper()} requires at least one argument", path
+                        )
+
                     if not args[0].is_string:
                         raise_config_error(
                             f"The variable name must be a string literal, '{args[0].sql()}' was given instead",
                             path,
                         )
-                    used_variables.add(args[0].this.lower())
+
+                    var_name = args[0].this.lower()
+                    used_variables[var_name] = _is_metadata_var(
+                        var_name, macro_func_or_var, is_metadata
+                    )
+                elif id(macro_func_or_var) not in visited_macro_funcs:
+                    # We only care about the top-level macro function calls to determine the metadata
+                    # status of the variables referenced in their ASTs. For example, in @m1(@m2(@x)),
+                    # if m1 is metadata-only but m2 is not, we can still determine that @x only affects
+                    # the metadata hash, since m2's result feeds into a metadata-only macro function.
+                    #
+                    # Generally, if the top-level call is known to be metadata-only or appear in a
+                    # metadata expression, then we can avoid traversing nested macro function calls.
+
+                    var_refs, _expr_under_metadata_macro_func, _visited_macro_funcs = (
+                        _extract_macro_func_variable_references(macro_func_or_var, is_metadata)
+                    )
+                    expr_under_metadata_macro_func.update(_expr_under_metadata_macro_func)
+                    visited_macro_funcs.update(_visited_macro_funcs)
+                    outermost_macro_func_ancestor_by_var |= {var_ref: name for var_ref in var_refs}
             elif macro_func_or_var.__class__ is d.MacroVar:
-                name = macro_func_or_var.name.lower()
-                if name in macros:
-                    # If this macro has been seen before as a non-metadata macro, prioritize that
-                    used_macros[name] = (
-                        macros[name],
-                        used_macros.get(name, (None, is_metadata))[1] and is_metadata,
+                var_name = macro_func_or_var.name.lower()
+                if var_name in macros:
+                    used_macros[var_name] = (
+                        macros[var_name],
+                        _is_metadata_macro(var_name, is_metadata),
+                    )
+                elif var_name in variables or var_name in blueprint_variables:
+                    used_variables[var_name] = _is_metadata_var(
+                        var_name, macro_func_or_var, is_metadata
                     )
-                elif name in variables:
-                    used_variables.add(name)
             elif (
                 isinstance(macro_func_or_var, (exp.Identifier, d.MacroStrReplace, d.MacroSQL))
             ) and "@" in macro_func_or_var.name:
@@ -104,12 +168,14 @@ def make_python_env(
                     macro_func_or_var.name
                 ):
                     var_name = braced_identifier or identifier
-                    if var_name in variables:
-                        used_variables.add(var_name)
+                    if var_name in variables or var_name in blueprint_variables:
+                        used_variables[var_name] = _is_metadata_var(
+                            var_name, macro_func_or_var, is_metadata
+                        )
 
     for macro_ref in jinja_macro_references or set():
         if macro_ref.package is None and macro_ref.name in macros:
-            used_macros[macro_ref.name] = (macros[macro_ref.name], None)
+            used_macros[macro_ref.name] = (macros[macro_ref.name], False)
 
     for name, (used_macro, is_metadata) in used_macros.items():
         if isinstance(used_macro, Executable):
@@ -131,16 +197,49 @@ def make_python_env(
         blueprint_variables=blueprint_variables,
         dialect=dialect,
         strict_resolution=strict_resolution,
+        outermost_macro_func_ancestor_by_var=outermost_macro_func_ancestor_by_var,
     )
 
 
+def _extract_macro_func_variable_references(
+    macro_func: exp.Expression,
+    is_metadata: bool,
+) -> t.Tuple[t.Set[str], t.Dict[int, bool], t.Set[int]]:
+    var_references = set()
+    visited_macro_funcs = set()
+    expr_under_metadata_macro_func = {}
+
+    for n in macro_func.walk():
+        if type(n) is d.MacroFunc:
+            visited_macro_funcs.add(id(n))
+
+            this = n.this
+            args = this.expressions
+
+            if this.name.lower() in (c.VAR, c.BLUEPRINT_VAR) and args and args[0].is_string:
+                var_references.add(args[0].this.lower())
+                expr_under_metadata_macro_func[id(n)] = is_metadata
+        elif isinstance(n, d.MacroVar):
+            var_references.add(n.name.lower())
+            expr_under_metadata_macro_func[id(n)] = is_metadata
+        elif isinstance(n, (exp.Identifier, d.MacroStrReplace, d.MacroSQL)) and "@" in n.name:
+            var_references.update(
+                (braced_identifier or identifier).lower()
+                for _, identifier, braced_identifier, _ in MacroStrTemplate.pattern.findall(n.name)
+            )
+            expr_under_metadata_macro_func[id(n)] = is_metadata
+
+    return (var_references, expr_under_metadata_macro_func, visited_macro_funcs)
+
+
 def _add_variables_to_python_env(
     python_env: t.Dict[str, Executable],
-    used_variables: t.Optional[t.Set[str]],
+    used_variables: t.Dict[str, t.Optional[bool]],
     variables: t.Optional[t.Dict[str, t.Any]],
     strict_resolution: bool = True,
     blueprint_variables: t.Optional[t.Dict[str, t.Any]] = None,
     dialect: DialectType = None,
+    outermost_macro_func_ancestor_by_var: t.Optional[t.Dict[str, str]] = None,
 ) -> t.Dict[str, Executable]:
     _, python_used_variables = parse_dependencies(
         python_env,
@@ -149,20 +248,67 @@ def _add_variables_to_python_env(
         variables=variables,
         blueprint_variables=blueprint_variables,
     )
-    used_variables = (used_variables or set()) | python_used_variables
+    for var_name, is_metadata in python_used_variables.items():
+        used_variables[var_name] = is_metadata and used_variables.get(var_name, True)
+
+    # Variables are treated as metadata-only when all of their references either:
+    # - appear in metadata-only expressions, such as `audits (...)`, virtual statements, etc
+    # - appear in the ASTs or definitions of metadata-only macros
+    #
+    # See also: https://github.com/TobikoData/sqlmesh/pull/4936#issuecomment-3136339936,
+    # specifically the "Terminology" and "Observations" section.
+    metadata_used_variables = {
+        var_name for var_name, is_metadata in used_variables.items() if is_metadata
+    }
+    for used_var, outermost_macro_func in (outermost_macro_func_ancestor_by_var or {}).items():
+        used_var_is_metadata = used_variables.get(used_var)
+        if used_var_is_metadata is False:
+            continue
+
+        # At this point we can decide whether a variable reference in a macro call's AST is
+        # metadata-only, because we've annotated the corresponding macro call in the python env.
+        if outermost_macro_func in python_env and python_env[outermost_macro_func].is_metadata:
+            metadata_used_variables.add(used_var)
+
+    non_metadata_used_variables = set(used_variables) - metadata_used_variables
+
+    if overlapping_variables := (non_metadata_used_variables & metadata_used_variables):
+        raise ConfigError(
+            f"Variables {', '.join(overlapping_variables)} are both metadata and non-metadata, "
+            "which is unexpected. Please file an issue at https://github.com/TobikoData/sqlmesh/issues/new."
+        )
+
+    metadata_variables = {
+        k: v for k, v in (variables or {}).items() if k in metadata_used_variables
+    }
+    variables = {k: v for k, v in (variables or {}).items() if k in non_metadata_used_variables}
 
-    variables = {k: v for k, v in (variables or {}).items() if k in used_variables}
     if variables:
         python_env[c.SQLMESH_VARS] = Executable.value(variables, sort_root_dict=True)
+    if metadata_variables:
+        python_env[c.SQLMESH_VARS_METADATA] = Executable.value(
+            metadata_variables, sort_root_dict=True, is_metadata=True
+        )
 
     if blueprint_variables:
+        metadata_blueprint_variables = {
+            k: SqlValue(sql=v.sql(dialect=dialect)) if isinstance(v, exp.Expression) else v
+            for k, v in blueprint_variables.items()
+            if k in metadata_used_variables
+        }
         blueprint_variables = {
             k.lower(): SqlValue(sql=v.sql(dialect=dialect)) if isinstance(v, exp.Expression) else v
             for k, v in blueprint_variables.items()
+            if k in non_metadata_used_variables
         }
-        python_env[c.SQLMESH_BLUEPRINT_VARS] = Executable.value(
-            blueprint_variables, sort_root_dict=True
-        )
+        if blueprint_variables:
+            python_env[c.SQLMESH_BLUEPRINT_VARS] = Executable.value(
+                blueprint_variables, sort_root_dict=True
+            )
+        if metadata_blueprint_variables:
+            python_env[c.SQLMESH_BLUEPRINT_VARS_METADATA] = Executable.value(
+                metadata_blueprint_variables, sort_root_dict=True, is_metadata=True
+            )
 
     return python_env
 
@@ -173,7 +319,7 @@ def parse_dependencies(
     strict_resolution: bool = True,
     variables: t.Optional[t.Dict[str, t.Any]] = None,
     blueprint_variables: t.Optional[t.Dict[str, t.Any]] = None,
-) -> t.Tuple[t.Set[str], t.Set[str]]:
+) -> t.Tuple[t.Set[str], t.Dict[str, bool]]:
     """
     Parses the source of a model function and finds upstream table dependencies
     and referenced variables based on calls to context / evaluator.
@@ -187,7 +333,8 @@ def parse_dependencies(
         blueprint_variables: The blueprint variables available to the python environment.
 
     Returns:
-        A tuple containing the set of upstream table dependencies and the set of referenced variables.
+        A tuple containing the set of upstream table dependencies and a mapping of
+        the referenced variables associated with their metadata status.
     """
 
     class VariableResolutionContext:
@@ -205,12 +352,16 @@ def parse_dependencies(
     local_env = dict.fromkeys(("context", "evaluator"), VariableResolutionContext)
 
     depends_on = set()
-    used_variables = set()
+    used_variables: t.Dict[str, bool] = {}
 
     for executable in python_env.values():
         if not executable.is_definition:
             continue
+
+        is_metadata = executable.is_metadata
         for node in ast.walk(ast.parse(executable.payload)):
+            next_variables = set()
+
             if isinstance(node, ast.Call):
                 func = node.func
                 if not isinstance(func, ast.Attribute) or not isinstance(func.value, ast.Name):
@@ -241,8 +392,11 @@ def parse_dependencies(
 
                 if func.value.id == "context" and func.attr in ("table", "resolve_table"):
                     depends_on.add(get_first_arg("model_name"))
-                elif func.value.id in ("context", "evaluator") and func.attr == c.VAR:
-                    used_variables.add(get_first_arg("var_name").lower())
+                elif func.value.id in ("context", "evaluator") and func.attr in (
+                    c.VAR,
+                    c.BLUEPRINT_VAR,
+                ):
+                    next_variables.add(get_first_arg("var_name").lower())
             elif (
                 isinstance(node, ast.Attribute)
                 and isinstance(node.value, ast.Name)
@@ -250,9 +404,9 @@ def parse_dependencies(
                 and node.attr == c.GATEWAY
             ):
                 # Check whether the gateway attribute is referenced.
-                used_variables.add(c.GATEWAY)
+                next_variables.add(c.GATEWAY)
             elif isinstance(node, ast.FunctionDef) and node.name == entrypoint:
-                used_variables.update(
+                next_variables.update(
                     [
                         arg.arg
                         for arg in [*node.args.args, *node.args.kwonlyargs]
@@ -260,6 +414,9 @@ def parse_dependencies(
                     ]
                 )
 
+            for var_name in next_variables:
+                used_variables[var_name] = used_variables.get(var_name, True) and bool(is_metadata)
+
     return depends_on, used_variables
 
 

sqlmesh/core/model/definition.py

@@ -1782,12 +1782,17 @@ class PythonModel(_Model):
         start, end = make_inclusive(start or c.EPOCH, end or c.EPOCH, self.dialect)
         execution_time = to_datetime(execution_time or c.EPOCH)
 
-        variables = env.get(c.SQLMESH_VARS, {})
-        variables.update(kwargs.pop("variables", {}))
-
+        variables = {
+            **env.get(c.SQLMESH_VARS, {}),
+            **env.get(c.SQLMESH_VARS_METADATA, {}),
+            **kwargs.pop("variables", {}),
+        }
         blueprint_variables = {
             k: d.parse_one(v.sql, dialect=self.dialect) if isinstance(v, SqlValue) else v
-            for k, v in env.get(c.SQLMESH_BLUEPRINT_VARS, {}).items()
+            for k, v in {
+                **env.get(c.SQLMESH_BLUEPRINT_VARS, {}),
+                **env.get(c.SQLMESH_BLUEPRINT_VARS_METADATA, {}),
+            }.items()
         }
         try:
             kwargs = {
@@ -1909,11 +1914,11 @@ def _extract_blueprint_variables(blueprint: t.Any, path: Path) -> t.Dict[str, t.
         return {}
     if isinstance(blueprint, (exp.Paren, exp.PropertyEQ)):
         blueprint = blueprint.unnest()
-        return {blueprint.left.name: blueprint.right}
+        return {blueprint.left.name.lower(): blueprint.right}
     if isinstance(blueprint, (exp.Tuple, exp.Array)):
-        return {e.left.name: e.right for e in blueprint.expressions}
+        return {e.left.name.lower(): e.right for e in blueprint.expressions}
     if isinstance(blueprint, dict):
-        return blueprint
+        return {k.lower(): v for k, v in blueprint.items()}
 
     raise_config_error(
         f"Expected a key-value mapping for the blueprint value, got '{blueprint}' instead",
@@ -2509,7 +2514,7 @@ def _create_model(
     if isinstance(getattr(kwargs.get("kind"), "merge_filter", None), exp.Expression):
         statements.append(kwargs["kind"].merge_filter)
 
-    jinja_macro_references, used_variables = extract_macro_references_and_variables(
+    jinja_macro_references, referenced_variables = extract_macro_references_and_variables(
         *(gen(e if isinstance(e, exp.Expression) else e[0]) for e in statements)
     )
 
@@ -2532,11 +2537,13 @@ def _create_model(
             _extract_migrated_dbt_variable_references(jinja_macros, variables)
         )
 
-        used_variables.update(nested_macro_used_variables)
+        referenced_variables.update(nested_macro_used_variables)
         variables.update(flattened_package_variables)
     else:
         for jinja_macro in jinja_macros.root_macros.values():
-            used_variables.update(extract_macro_references_and_variables(jinja_macro.definition)[1])
+            referenced_variables.update(
+                extract_macro_references_and_variables(jinja_macro.definition)[1]
+            )
 
     # Merge model-specific audits with default audits
     if default_audits := defaults.pop("audits", None):
@@ -2598,7 +2605,7 @@ def _create_model(
         module_path,
         macros or macro.get_registry(),
         variables=variables,
-        used_variables=used_variables,
+        referenced_variables=referenced_variables,
         path=path,
         python_env=python_env,
         strict_resolution=depends_on is None,

sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py

@@ -0,0 +1,74 @@
+"""
+This script's goal is to warn users about two situations that could lead to a diff:
+
+- They have blueprint models and some of their variables may be trimmed from `python_env`
+- Variables are used in metadata-only contexts, e.g., within metadata-only macros
+
+Context:
+
+We used to store *all* blueprint variables in `python_env`, even though some of them were
+redundant. For example, if a blueprint variable is only used in the model's `name` property,
+then it is rendered once, at load time, and after that point it's not needed elsewhere.
+
+This behavior is now different: we only store the blueprint variables that are required to render
+expressions at runtime, such as model query or runtime-rendered properties, like `merge_filter`.
+
+Additionally, variables were previously treated as non-metadata, regardless of how they were used.
+This behavior changed as well: SQLMesh now analyzes variable references and tracks the data flow,
+in order to detect whether changing them will result in a metadata diff for a given model.
+
+Some examples where variables can be treated as metadata-only `python_env` executables are:
+
+- A variable is referenced in metadata-only macros
+- A variable is referenced in metadata-only expressions, such as virtual update statements
+- A variable is passed as argument to metadata-only macros
+"""
+
+import json
+
+from sqlglot import exp
+
+from sqlmesh.core.console import get_console
+
+SQLMESH_VARS = "__sqlmesh__vars__"
+SQLMESH_BLUEPRINT_VARS = "__sqlmesh__blueprint__vars__"
+METADATA_HASH_EXPRESSIONS = {"on_virtual_update", "audits", "signals", "audit_definitions"}
+
+
+def migrate(state_sync, **kwargs):  # type: ignore
+    engine_adapter = state_sync.engine_adapter
+    schema = state_sync.schema
+    snapshots_table = "_snapshots"
+    if schema:
+        snapshots_table = f"{schema}.{snapshots_table}"
+
+    warning = (
+        "SQLMesh detected that it may not be able to fully migrate the state database. This should not impact "
+        "the migration process, but may result in unexpected changes being reported by the next `sqlmesh plan` "
+        "command. Please run `sqlmesh diff prod` after the migration has completed, before making any new "
+        "changes. If any unexpected changes are reported, consider running a forward-only plan to apply these "
+        "changes and avoid unnecessary backfills: sqlmesh plan prod --forward-only. "
+        "See https://sqlmesh.readthedocs.io/en/stable/concepts/plans/#forward-only-plans for more details.\n"
+    )
+
+    for (snapshot,) in engine_adapter.fetchall(
+        exp.select("snapshot").from_(snapshots_table), quote_identifiers=True
+    ):
+        parsed_snapshot = json.loads(snapshot)
+        node = parsed_snapshot["node"]
+
+        # Standalone audits don't have a data hash, so they're unaffected
+        if node.get("source_type") == "audit":
+            continue
+
+        python_env = node.get("python_env") or {}
+
+        if SQLMESH_BLUEPRINT_VARS in python_env or (
+            SQLMESH_VARS in python_env
+            and (
+                any(v.get("is_metadata") for v in python_env.values())
+                or any(node.get(k) for k in METADATA_HASH_EXPRESSIONS)
+            )
+        ):
+            get_console().log_warning(warning)
+            return

sqlmesh/utils/jinja.py

@@ -229,7 +229,7 @@ def extract_macro_references_and_variables(
                     )
 
         for call_name, node in extract_call_names(jinja_str):
-            if call_name[0] == c.VAR:
+            if call_name[0] in (c.VAR, c.BLUEPRINT_VAR):
                 assert isinstance(node, nodes.Call)
                 args = [jinja_call_arg_name(arg) for arg in node.args]
                 if args and args[0]:

sqlmesh/utils/metaprogramming.py

@@ -283,7 +283,7 @@ def build_env(
     env: t.Dict[str, t.Tuple[t.Any, t.Optional[bool]]],
     name: str,
     path: Path,
-    is_metadata_obj: t.Optional[bool] = None,
+    is_metadata_obj: bool = False,
 ) -> None:
     """Fills in env dictionary with all globals needed to execute the object.
 
@@ -299,7 +299,7 @@ def build_env(
     # We don't rely on `env` to keep track of visited objects, because it's populated in post-order
     visited: t.Set[str] = set()
 
-    def walk(obj: t.Any, name: str, is_metadata: t.Optional[bool] = None) -> None:
+    def walk(obj: t.Any, name: str, is_metadata: bool = False) -> None:
         obj_module = inspect.getmodule(obj)
         if obj_module and obj_module.__name__ == "builtins":
             return
@@ -320,7 +320,7 @@ def build_env(
                 # The existing object in the env is "metadata only" but we're walking it again as a
                 # non-"metadata only" dependency, so we update this flag to ensure all transitive
                 # dependencies are also not marked as "metadata only"
-                is_metadata = None
+                is_metadata = False
 
             if hasattr(obj, c.SQLMESH_MACRO):
                 # We only need to add the undecorated code of @macro() functions in env, which
@@ -380,7 +380,7 @@ def build_env(
             )
 
     # The "metadata only" annotation of the object is transitive
-    walk(obj, name, is_metadata_obj or getattr(obj, c.SQLMESH_METADATA, None))
+    walk(obj, name, is_metadata_obj or getattr(obj, c.SQLMESH_METADATA, False))
 
 
 @dataclass
@@ -432,7 +432,11 @@ class Executable(PydanticModel):
         cls, v: t.Any, is_metadata: t.Optional[bool] = None, sort_root_dict: bool = False
     ) -> Executable:
         payload = _dict_sort(v) if sort_root_dict else repr(v)
-        return Executable(payload=payload, kind=ExecutableKind.VALUE, is_metadata=is_metadata)
+        return Executable(
+            payload=payload,
+            kind=ExecutableKind.VALUE,
+            is_metadata=is_metadata or None,
+        )
 
 
 def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable]:
@@ -447,6 +451,9 @@ def serialize_env(env: t.Dict[str, t.Any], path: Path) -> t.Dict[str, Executable
     serialized = {}
 
     for k, (v, is_metadata) in env.items():
+        # We don't store `False` for `is_metadata` to reduce the pydantic model's payload size
+        is_metadata = is_metadata or None
+
         if isinstance(v, LITERALS) or v is None:
             serialized[k] = Executable.value(v, is_metadata=is_metadata)
         elif inspect.ismodule(v):

{sqlmesh-0.209.1.dev12.dist-info → sqlmesh-0.209.1.dev13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlmesh
-Version: 0.209.1.dev12
+Version: 0.209.1.dev13
 Summary: Next-generation data transformation framework
 Author-email: "TobikoData Inc." <engineering@tobikodata.com>
 License:                                  Apache License

{sqlmesh-0.209.1.dev12.dist-info → sqlmesh-0.209.1.dev13.dist-info}/RECORD

@@ -1,5 +1,5 @@
 sqlmesh/__init__.py,sha256=QjwaamJIdW7voXoS880yAx7tHm-Im4_mqGA-XJgafos,7631
-sqlmesh/_version.py,sha256=7OqlZXPJ-sO1ZD5h7xeYT3AcKstkLcz3pDByvyZckQs,723
+sqlmesh/_version.py,sha256=Z-QEmLpn7WCZTovKI1S7q9PMPW5I4VTOb_KNgW5Qf5Y,723
 sqlmesh/magics.py,sha256=bEzYqXHcM0P3CjyUZ1O9arlmXAdLAhoglKbQ29fD9AE,41842
 sqlmesh/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/cicd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -12,14 +12,14 @@ sqlmesh/cli/project_init.py,sha256=5JmDmUK0WJs39hM-p76uzHYdo1Hnx6fKuJg_pR3Hn5A,1
 sqlmesh/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/core/_typing.py,sha256=e1f7xO2aZCy38M_3VoDy-XI3shyd99-IFctgIFSgXWM,497
 sqlmesh/core/console.py,sha256=KbIXoxt5hpDENEolY-DpBlPmcWtJ9vzEgF52mjG2jPE,165710
-sqlmesh/core/constants.py,sha256=gUABXWYgH_XOSbaZAJsR5o1dmA61JDCvBDyxqqXX3LM,2630
+sqlmesh/core/constants.py,sha256=Wg6qApCRhENA1me7ELUS5GnArKlTBH-8192gQYKQnE8,2759
 sqlmesh/core/context.py,sha256=bFAvnjg3YENIgWVDYfflKA5yW6o6gMEZhiSKBPIrBk8,127613
 sqlmesh/core/context_diff.py,sha256=_Ur55fGLJ58Icsp44I4pVhdi7_JHiHq45mWIco9oFzk,21792
 sqlmesh/core/dialect.py,sha256=wJ2cDoxz72LIRJ3iBZdCqtnKMbMUdIftl-AvbVldI28,52413
 sqlmesh/core/environment.py,sha256=odus40R-YzdQdS_IiUq_tFouUXIhnpwmNwo8dE8Aw1k,13041
 sqlmesh/core/lineage.py,sha256=zjB0Zfamo2Fja2r5SSZPMbrEKTXr1WjozZPVqvxdybI,3143
 sqlmesh/core/loader.py,sha256=z8KNT5igXc_g-5X1BPPL9XAbpve3ZQp7OkkX63iT0Gc,41179
-sqlmesh/core/macros.py,sha256=_tQohZfLoIjO7qpgNu91Qqu3GRJ22k4Ic1AVdowTHuY,62542
+sqlmesh/core/macros.py,sha256=G1TD1Lv-iTNOh1JtHwRFOrnThqDBFuAQgQJtp6UDe0I,63023
 sqlmesh/core/node.py,sha256=3ChBz1PIMJhp-jzed_GLBrtmEF2v_BVDEM5WNe0ssPk,14658
 sqlmesh/core/notification_target.py,sha256=PPGoDrgbRKxr27vJEu03XqNTQLYTw0ZF_b0yAapxGeI,16158
 sqlmesh/core/reference.py,sha256=k7OSkLqTjPR8WJjNeFj0xAJ297nZUMgb_iTVwKRRKjc,4875
@@ -36,7 +36,7 @@ sqlmesh/core/analytics/collector.py,sha256=sggJvER16Jrg5triTC27E2KwaQvfkVDv4D6R1
 sqlmesh/core/analytics/dispatcher.py,sha256=SQw5tHIETb7ei_oCj7ktHEJyeJNbKyIYiOPxQXyQuIg,6259
 sqlmesh/core/audit/__init__.py,sha256=3jogKe-G4X0rOxwH5QVuRRioyesQffSWxKU4PVwMPvA,216
 sqlmesh/core/audit/builtin.py,sha256=adA-cFCphB66jVY4n66cbwbRTFF1FTEIMrLhRDpbBo0,17783
-sqlmesh/core/audit/definition.py,sha256=QXXObhR_jHgjWLu0qXxPuIXm0GlFQq33rhp11m_xAUU,18306
+sqlmesh/core/audit/definition.py,sha256=iPpGEOwFIXFHQFQGMyapCm7jnlotwMa-gNaCgTwzlIA,18360
 sqlmesh/core/config/__init__.py,sha256=5zxTXjUMuxBj4XlX6bmpIWTp4GsZY5rIdKSCTYbxRBM,2070
 sqlmesh/core/config/base.py,sha256=t8NQmsgQoZSc-k0dlDiCb8t1jj0AMYdGZ-6se9q_Pks,4898
 sqlmesh/core/config/categorizer.py,sha256=6vzUoNLjR6GOEb_2mYVz2TwmMv2BfldgHX2u-Le5HZs,1975
@@ -86,9 +86,9 @@ sqlmesh/core/metric/definition.py,sha256=Yd5aVgsZCDPJ43aGP7WqtzZOuuSUtB8uJGVA6Jw
 sqlmesh/core/metric/rewriter.py,sha256=GiSTHfn2kinqCfNPYgZPRk93JFLzVaaejHtHDQ0yXZI,7326
 sqlmesh/core/model/__init__.py,sha256=C8GRZ53xuXEA9hQv3BQS9pNNyd9rZ06R_B96UYGhDu4,1644
 sqlmesh/core/model/cache.py,sha256=csun0RJguHzKX6-qITcOs4fVP4f8_Ts8qiUVV4sHY6Q,7869
-sqlmesh/core/model/common.py,sha256=Gt1nBdbILfR52Rhq2KxF0EMGkB3J5zfk6UH7ecLpMZI,17969
+sqlmesh/core/model/common.py,sha256=xCMdwkJXnwjE9-56wNb1HiER0KHmWbBzc2Tm--2u168,25610
 sqlmesh/core/model/decorator.py,sha256=wUUg5t0K559tV9Pqt6FE2EWXt5TPGb4wF2Yg4B4hX4w,9251
-sqlmesh/core/model/definition.py,sha256=zDezkLS4FAWkd5OKSTtk8VGmepirq-zGSl4aBsLehLg,113573
+sqlmesh/core/model/definition.py,sha256=uouwIbWA-m2cFXxvDrBVHKxR0ycSAq8Rz3Qv3xY7ewA,113852
 sqlmesh/core/model/kind.py,sha256=uN73ycANfM94CxcOdkUTclLansNvwo3wySkkl--QzY0,36322
 sqlmesh/core/model/meta.py,sha256=IE-b_OkCcKzXuQBsopiaf0l5a5Ay903N_64Z1s4cXGk,20064
 sqlmesh/core/model/schema.py,sha256=_HMYfzK9wWXh7_CQDIIGnuQUD4aiX3o5D2cRp2sERzc,3387
@@ -264,6 +264,7 @@ sqlmesh/migrations/v0084_normalize_quote_when_matched_and_merge_filter.py,sha256
 sqlmesh/migrations/v0085_deterministic_repr.py,sha256=mkbZV0rzXJ5bmN1Y75kFI9T6gbnx_sy-zn_UzbOL1Sg,4265
 sqlmesh/migrations/v0086_check_deterministic_bug.py,sha256=WiYcmhQD434GXWiiakQtcy9ESmB35spXMltV43nvU7Q,2953
 sqlmesh/migrations/v0087_normalize_blueprint_variables.py,sha256=GI6KKA-6PtONUoOnyvLpRTUmPjbpa-CKhDa0iqJW01U,4421
+sqlmesh/migrations/v0088_warn_about_variable_python_env_diffs.py,sha256=zCLT6Yqwny73SlfACWjO1bZfs8lZ7gJBov5Iz4akwiQ,3254
 sqlmesh/schedulers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlmesh/utils/__init__.py,sha256=51VqA2etTZ6kX54250s1hN5ui7D89OeUTUQ9kgMze7s,11554
 sqlmesh/utils/aws.py,sha256=krxyhrKtsMJDEjT567UbGZogjJ9JRHkvAV-Cg3FVndM,1062
@@ -278,9 +279,9 @@ sqlmesh/utils/date.py,sha256=uS0Kii4hFJeTXqXtM39sDCzvFhTAoPk_NPg51T1YYeo,16218
 sqlmesh/utils/errors.py,sha256=8FE33Eo3KN3Y4s5EZM5t92jZi9BAVGzkr3QCGlrLeDk,5679
 sqlmesh/utils/git.py,sha256=v1MD4Zwn52UBn_tTfoKn8SAPJVGC6SEFrJ4WEJFbgF0,1868
 sqlmesh/utils/hashing.py,sha256=nZRKvLNQ83tLG4IoXshVJZf-MbDrXC1HOeNw8Ji-tMM,578
-sqlmesh/utils/jinja.py,sha256=zulSPIICdFOjN__xkAiMpxbCS7S0--xAFK9d8VG0KvU,27844
+sqlmesh/utils/jinja.py,sha256=7IPOu4pjUmxX_ZNL7S0UN23CXJM8EUzPaFt3Wzr1rxI,27863
 sqlmesh/utils/lineage.py,sha256=zz9BPc6MShRy9LEXmAp02x6oKt4ubVNUPdapFVFKkac,16019
-sqlmesh/utils/metaprogramming.py,sha256=yDWS3_tkm6Yd5TNHgii2x_VdZ2pVM0vL6oFuwFMMs0A,23554
+sqlmesh/utils/metaprogramming.py,sha256=6NQKi9PpYhgJEu0Wr_TztRZZ2Kog4Dic9syNqbvztMw,23727
 sqlmesh/utils/migration.py,sha256=R7Ck6FX-LfcujGy7sgS0kDWlK-xAEBODr3gqIQ8gC08,835
 sqlmesh/utils/pandas.py,sha256=FytTSLcRNtO0-YMvpoDBkkut3yoQJR9mBSKpif8lxug,2497
 sqlmesh/utils/process.py,sha256=vpXcpJ1eIbivs7gZdwTKSJI7OS6kKBebnaBzz75ykvs,2428
@@ -288,7 +289,7 @@ sqlmesh/utils/pydantic.py,sha256=o_NsXbIpDqNpUA1Uc5xF0ZzoXQYYB0DfHwdRxBwXPNk,120
 sqlmesh/utils/rich.py,sha256=cwQ5nJ6sgz64xHtoh6_ec7ReV5YpsOGhMtUJnwoRfEI,3549
 sqlmesh/utils/windows.py,sha256=r7Yin4WEwbvH7sKA3hOJtRsclmscNuBeCklCfVPvtQs,611
 sqlmesh/utils/yaml.py,sha256=hvo5lmybYLcIx4p6CQwufS8X4MbwS60lEBnCmYJCXmU,2336
-sqlmesh-0.209.1.dev12.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
+sqlmesh-0.209.1.dev13.dist-info/licenses/LICENSE,sha256=OlMefUjgWJdULtf84BLW0AZZcY8DwdgQqb_1j2862j8,11346
 sqlmesh_dbt/__init__.py,sha256=awYS5y5mz-1NUmx6i5h5NSTJ7tidRl9NC0FAnFWSF6U,350
 sqlmesh_dbt/cli.py,sha256=Ekn77WLMmyyr31aoDoWrZp5DS0rfiddy7pNohBK8G0s,2115
 sqlmesh_dbt/console.py,sha256=1B0B5_R1DlrEkyWVDl8G78-B6_K-ZtLPsQEhqPCfLEU,188
@@ -410,8 +411,8 @@ web/server/api/endpoints/models.py,sha256=kwj0s7uve3iZSMfmjkoPVMFMeY1sD0peTeyrWf
 web/server/api/endpoints/modules.py,sha256=8hqqgonGay_mJmpCw0IdbjsPhWlQH2VLdKAqha-myac,468
 web/server/api/endpoints/plan.py,sha256=bbbY50W_2MsZSTxOHWMKz0tbIm75nsRSlPy8GI2fg9Q,9306
 web/server/api/endpoints/table_diff.py,sha256=SCy3-pr5x6bEEiOaZGCDbiHQhpgxwmnz3Acqa2Ie6jI,2090
-sqlmesh-0.209.1.dev12.dist-info/METADATA,sha256=M3xduNBEn62vHm2RLj8dwbolcTWTz77Rk0Mlu3z_tOY,26686
-sqlmesh-0.209.1.dev12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sqlmesh-0.209.1.dev12.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
-sqlmesh-0.209.1.dev12.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
-sqlmesh-0.209.1.dev12.dist-info/RECORD,,
+sqlmesh-0.209.1.dev13.dist-info/METADATA,sha256=xyA8vDp0_wasj_KFmp6AZoegsW7YBPLiaOnzE_ndzf4,26686
+sqlmesh-0.209.1.dev13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sqlmesh-0.209.1.dev13.dist-info/entry_points.txt,sha256=sHAf6tQczIM8xZoduN4qaUjV7QEPVUUW_LCT8EDUMv4,155
+sqlmesh-0.209.1.dev13.dist-info/top_level.txt,sha256=RQ-33FPe2IgL0rgossAfJkCRtqslz9b7wFARqiWLC5Q,24
+sqlmesh-0.209.1.dev13.dist-info/RECORD,,