dbt-autofix 0.0.0.post1.dev0__py3-none-any.whl

dbt_autofix/_jinja_environment.py

@@ -0,0 +1,305 @@
+"""Vendored Jinja environment setup from dbt-common.
+
+This module contains the minimal subset of dbt-common's Jinja machinery needed
+to parse (not render) dbt templates. It exists to avoid pulling in the full
+dbt-common transitive dependency tree (agate, protobuf, mashumaro, etc.) for
+a parse-only use case.
+
+Vendored from:
+  - dbt_common/utils/jinja.py  (name-mangling helpers)
+  - dbt_common/clients/jinja.py (parser, environment, extensions, undefined)
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Callable, ClassVar, Dict, List, Literal, NoReturn, Optional, Type, Union
+
+import jinja2
+import jinja2.ext
+import jinja2.nodes
+import jinja2.parser
+import jinja2.sandbox
+from jinja2 import select_autoescape
+
+# ---------------------------------------------------------------------------
+# Constants (from dbt_common/utils/jinja.py)
+# https://github.com/dbt-labs/dbt-common/blob/5b331b9c50ca5fee959a9e4fa9ecca964549930c/dbt_common/utils/jinja.py#L6
+# ---------------------------------------------------------------------------
+
+MACRO_PREFIX = "dbt_macro__"
+DOCS_PREFIX = "dbt_docs__"
+
+
+# ---------------------------------------------------------------------------
+# Name-prefixing helpers (from dbt_common/utils/jinja.py)
+# https://github.com/dbt-labs/dbt-common/blob/5b331b9c50ca5fee959a9e4fa9ecca964549930c/dbt_common/utils/jinja.py#L10
+# ---------------------------------------------------------------------------
+
+
+def get_dbt_macro_name(name: str) -> str:
+    if name is None:
+        raise ValueError("Got None for a macro name!")
+    return f"{MACRO_PREFIX}{name}"
+
+
+def get_materialization_macro_name(
+    materialization_name: str,
+    adapter_type: Optional[str] = None,
+) -> str:
+    """Upstream ``with_prefix`` param dropped; always prefixes (the only usage in jinja client code)."""
+    if adapter_type is None:
+        adapter_type = "default"
+    name = f"materialization_{materialization_name}_{adapter_type}"
+    return get_dbt_macro_name(name)
+
+
+def get_docs_macro_name(docs_name: str) -> str:
+    """Upstream ``with_prefix`` param dropped; always prefixes (the only usage in jinja client code)."""
+    return f"{DOCS_PREFIX}{docs_name}"
+
+
+def get_test_macro_name(test_name: str) -> str:
+    """Upstream ``with_prefix`` param dropped; always prefixes (the only usage in jinja client code)."""
+    name = f"test_{test_name}"
+    return get_dbt_macro_name(name)
+
+
+# ---------------------------------------------------------------------------
+# MacroType / MacroFuzzParser / MacroFuzzEnvironment
+# (from dbt_common/clients/jinja.py)
+# https://github.com/dbt-labs/dbt-common/blob/5b331b9c50ca5fee959a9e4fa9ecca964549930c/dbt_common/clients/jinja.py#L101
+# ---------------------------------------------------------------------------
+
+
+@dataclasses.dataclass
+class MacroType:
+    name: str
+    type_params: List[MacroType] = dataclasses.field(default_factory=list)
+
+
+_ParseReturn = Union[jinja2.nodes.Node, List[jinja2.nodes.Node]]
+
+
+class MacroFuzzParser(jinja2.parser.Parser):
+    def parse_macro(self) -> jinja2.nodes.Macro:
+        node = jinja2.nodes.Macro(lineno=next(self.stream).lineno)
+        node.name = get_dbt_macro_name(self.parse_assign_target(name_only=True).name)
+        self.parse_signature(node)
+        node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
+        return node
+
+    def parse_signature(self, node: Union[jinja2.nodes.Macro, jinja2.nodes.CallBlock]) -> None:
+        setattr(node, "arg_types", [])
+        setattr(node, "has_type_annotations", False)
+
+        args = node.args = []
+        defaults = node.defaults = []
+
+        self.stream.expect("lparen")
+        while self.stream.current.type != "rparen":
+            if args:
+                self.stream.expect("comma")
+
+            arg = self.parse_assign_target(name_only=True)
+            arg.set_ctx("param")
+
+            # TODO: Naming and typing is odd here. Probably better as macro_type: Optional[MacroType]
+            type_name: Union[MacroType, Literal[""]]
+
+            if self.stream.skip_if("colon"):
+                node.has_type_annotations = True  # ty:ignore[invalid-assignment]
+                type_name = self.parse_type_name()
+            else:
+                type_name = ""
+
+            node.arg_types.append(type_name)  # type: ignore
+
+            if self.stream.skip_if("assign"):
+                defaults.append(self.parse_expression())
+            elif defaults:
+                self.fail("non-default argument follows default argument")
+
+            args.append(arg)
+        self.stream.expect("rparen")
+
+    def parse_type_name(self) -> MacroType:
+        type_name = self.stream.expect("name").value
+        type_ = MacroType(type_name)
+
+        if self.stream.skip_if("lbracket"):
+            while self.stream.current.type != "rbracket":
+                if type_.type_params:
+                    self.stream.expect("comma")
+                param_type = self.parse_type_name()
+                type_.type_params.append(param_type)
+
+            self.stream.expect("rbracket")
+
+        return type_
+
+
+class MacroFuzzEnvironment(jinja2.sandbox.SandboxedEnvironment):
+    def _parse(self, source: str, name: Optional[str], filename: Optional[str]) -> jinja2.nodes.Template:
+        return MacroFuzzParser(self, source, name, filename).parse()
+
+
+# ---------------------------------------------------------------------------
+# Extensions (from dbt_common/clients/jinja.py)
+# https://github.com/dbt-labs/dbt-common/blob/5b331b9c50ca5fee959a9e4fa9ecca964549930c/dbt_common/clients/jinja.py#L430
+# ---------------------------------------------------------------------------
+
+SUPPORTED_LANG_ARG = jinja2.nodes.Name("supported_languages", "param")
+
+
+class MaterializationExtension(jinja2.ext.Extension):
+    tags: ClassVar[List[str]] = ["materialization"]
+
+    def parse(self, parser: jinja2.parser.Parser) -> _ParseReturn:
+        node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
+        materialization_name = parser.parse_assign_target(name_only=True).name
+
+        adapter_name = "default"
+        node.args = []
+        node.defaults = []
+
+        while parser.stream.skip_if("comma"):
+            target = parser.parse_assign_target(name_only=True)
+
+            if target.name == "default":
+                pass
+            elif target.name == "adapter":
+                parser.stream.expect("assign")
+                value = parser.parse_expression()
+                adapter_name = value.value  # ty: ignore[unresolved-attribute]
+            elif target.name == "supported_languages":
+                target.set_ctx("param")
+                node.args.append(target)
+                parser.stream.expect("assign")
+                languages = parser.parse_expression()
+                node.defaults.append(languages)
+            else:
+                # Upstream raises MaterializationArgError (a dbt-common exception).
+                # We use ValueError to avoid the dbt-common dependency.
+                raise ValueError(f"Unexpected argument '{target.name}' to materialization '{materialization_name}'")
+
+        if SUPPORTED_LANG_ARG not in node.args:
+            node.args.append(SUPPORTED_LANG_ARG)
+            node.defaults.append(jinja2.nodes.List([jinja2.nodes.Const("sql")]))
+
+        node.name = get_materialization_macro_name(materialization_name, adapter_name)
+        node.body = parser.parse_statements(("name:endmaterialization",), drop_needle=True)
+        return node
+
+
+class DocumentationExtension(jinja2.ext.Extension):
+    tags: ClassVar[List[str]] = ["docs"]
+
+    def parse(self, parser: jinja2.parser.Parser) -> _ParseReturn:
+        node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
+        docs_name = parser.parse_assign_target(name_only=True).name
+
+        node.args = []
+        node.defaults = []
+        node.name = get_docs_macro_name(docs_name)
+        node.body = parser.parse_statements(("name:enddocs",), drop_needle=True)
+        return node
+
+
+class TestExtension(jinja2.ext.Extension):
+    tags: ClassVar[List[str]] = ["test"]
+
+    def parse(self, parser: jinja2.parser.Parser) -> _ParseReturn:
+        node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
+        test_name = parser.parse_assign_target(name_only=True).name
+
+        parser.parse_signature(node)
+        node.name = get_test_macro_name(test_name)
+        node.body = parser.parse_statements(("name:endtest",), drop_needle=True)
+        return node
+
+
+# ---------------------------------------------------------------------------
+# Undefined handling (from dbt_common/clients/jinja.py)
+# ---------------------------------------------------------------------------
+def _is_dunder_name(name: str) -> bool:
+    return name.startswith("__") and name.endswith("__")
+
+
+def create_undefined() -> Type[jinja2.Undefined]:
+    class Undefined(jinja2.Undefined):
+        def __init__(
+            self,
+            hint: Optional[str] = None,
+            obj: Any = None,
+            name: Optional[str] = None,
+            exc: Any = None,
+        ) -> None:
+            super().__init__(hint=hint, name=name)
+            self.name = name
+            self.hint = hint
+            self.unsafe_callable = False
+            self.alters_data = False
+
+        def __getitem__(self, name: Any) -> Undefined:
+            return self
+
+        def __getattr__(self, name: str) -> Undefined:
+            if name == "name" or _is_dunder_name(name):
+                raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
+            self.name = name
+            return self.__class__(hint=self.hint, name=self.name)
+
+        def __call__(self, *args: Any, **kwargs: Any) -> Undefined:
+            return self
+
+        def __reduce__(self) -> NoReturn:
+            raise TypeError(f"Compilation Error: undefined variable '{self.name or 'unknown'}'")
+
+    return Undefined
+
+
+# ---------------------------------------------------------------------------
+# Filters (from dbt_common/clients/jinja.py)
+# ---------------------------------------------------------------------------
+def is_list(value: Any) -> bool:
+    return isinstance(value, list)
+
+
+TEXT_FILTERS: Dict[str, Callable[[Any], Any]] = {
+    "as_text": lambda x: x,
+    "as_bool": lambda x: x,
+    "as_native": lambda x: x,
+    "as_number": lambda x: x,
+    "is_list": is_list,
+}
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+def get_jinja_environment() -> jinja2.Environment:
+    """Return a Jinja environment configured for parsing dbt templates.
+
+    This is equivalent to ``get_environment(None, capture_macros=True)``
+    from dbt-common but without the dbt-common dependency.
+    """
+    args: Dict[str, Any] = {
+        "extensions": [
+            "jinja2.ext.do",
+            "jinja2.ext.loopcontrols",
+            MaterializationExtension,
+            DocumentationExtension,
+            TestExtension,
+        ],
+        "undefined": create_undefined(),
+        # A difference from dbt-common's implementation
+        # This disables autoescaping when rendering HTML / XML templates, to improve security posture
+        # But it should be irrelevant, as we only parse templates in autofix, and those should always be sql and yaml
+        # But let's prefer secure settings for prevention
+        "autoescape": select_autoescape(),
+    }
+
+    env = MacroFuzzEnvironment(**args)
+    env.filters.update(TEXT_FILTERS)
+    return env

dbt_autofix/dbt_api.py

@@ -0,0 +1,343 @@
+import json
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+import httpx
+from rich.console import Console
+
+console = Console()
+HTTP_ERROR_CODE = 400
+
+
+def job_dict_to_payload(job_dict: dict) -> dict:
+    """Convert a job dictionary to a payload dictionary."""
+
+    fields_to_remove = {
+        "raw_dbt_version",
+        "created_at",
+        "updated_at",
+        "deactivated",
+        "run_failure_count",
+        "lifecycle_webhooks",
+        "lifecycle_webhooks_url",
+        "is_deferrable",
+        "generate_sources",
+        "cron_humanized",
+        "next_run",
+        "next_run_humanized",
+        "is_system",
+        "account",
+        "project",
+        "environment",
+        "most_recent_run",
+        "most_recent_completed_run",
+        "identifier",
+    }
+
+    payload = {}
+    for key, value in job_dict.items():
+        if key not in fields_to_remove:
+            payload[key] = value
+
+    return payload
+
+
+def job_steps_updated(job_dict: dict, behavior_change: bool) -> tuple[bool, List[str]]:
+    """Check if the job steps need to be updated."""
+
+    exec_steps = job_dict.get("execute_steps", [])
+
+    # Create a copy of the steps to avoid modifying the original
+    updated_steps = exec_steps.copy()
+    steps_changed = False
+
+    if behavior_change:
+        update_step_rules = [
+            step_remove_source_freshness_output,
+        ]
+    else:
+        update_step_rules = [
+            step_regex_replace_m_with_s,
+        ]
+
+    for i, step in enumerate(updated_steps):
+        if isinstance(step, str):
+            for update_step_fn in update_step_rules:
+                new_step = update_step_fn(step)
+
+                if new_step != step:
+                    steps_changed = True
+                    updated_steps[i] = new_step
+
+    return steps_changed, updated_steps
+
+
+def step_regex_replace_m_with_s(step: str) -> str:
+    """Replace -m with -s and --model/--models with --select."""
+    step = re.sub(r"(\s)-m(\s)", r"\1-s\2", step)
+    step = re.sub(r"(\s)--model[s]?(\s)", r"\1--select\2", step)
+    return step
+
+
+def step_remove_source_freshness_output(step: str) -> str:
+    """Remove --output in source freshness commands."""
+    if ("dbt source freshness") in step:
+        step = re.sub(r"(\s)-o(\s+)\S+", "", step)
+        step = re.sub(r"(\s)--output(\s+)\S+", "", step)
+    return step
+
+
+class DBTClient:
+    """A minimalistic API client for fetching dbt data via the Admin API."""
+
+    def __init__(
+        self,
+        account_id: int,
+        api_key: Optional[str],
+        base_url: str = "https://cloud.getdbt.com",
+        disable_ssl_verification: bool = False,
+    ) -> None:
+        self.account_id = account_id
+        self._api_key = api_key
+
+        self.base_url = base_url
+        self._headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+            "User-Agent": "dbt-autofix",
+        }
+        self._verify = not disable_ssl_verification
+        if not self._verify:
+            self._client = httpx.Client(verify=False)
+        else:
+            self._client = httpx.Client()
+
+    def update_job(self, job: dict) -> dict:
+        """Update an existing dbt Cloud job using a new JobDefinition"""
+
+        logging.debug(f"Updating {job['name']}")
+
+        response = self._client.post(
+            url=f"{self.base_url}/api/v2/accounts/{self.account_id}/jobs/{job['id']}/",
+            headers=self._headers,
+            json=job_dict_to_payload(job),
+        )
+
+        if response.status_code >= HTTP_ERROR_CODE:
+            logging.error(response.json())
+            raise Exception(f"Error updating job {job['name']} - {response.json()}")
+        else:
+            logging.info(f"Job '{job['id']} - {job['name']}' updated successfully.")
+
+        return response.json()["data"]
+
+    def get_jobs(
+        self,
+        project_ids: Optional[List[int]] = None,
+        environment_ids: Optional[List[int]] = None,
+    ) -> List[dict]:
+        """Return a list of Jobs for all the dbt Cloud jobs in an environment."""
+
+        self._check_for_creds()
+        project_ids = project_ids or []
+        environment_ids = environment_ids or []
+
+        jobs: List[dict] = []
+        if len(environment_ids) > 1:
+            for env_id in environment_ids:
+                jobs.extend(self._fetch_jobs(project_ids, env_id))
+        elif len(environment_ids) == 1:
+            jobs = self._fetch_jobs(project_ids, environment_ids[0])
+        else:
+            jobs = self._fetch_jobs(project_ids, None)
+
+        return jobs
+
+    def _fetch_jobs(self, project_ids: List[int], environment_id: Optional[int]) -> List[dict]:
+        offset = 0
+        jobs: List[dict] = []
+
+        while True:
+            parameters = self._build_parameters(project_ids, environment_id, offset)
+            job_data = self._make_request(parameters)
+
+            if not job_data:
+                return []
+
+            jobs.extend(job_data["data"])
+
+            if (
+                job_data["extra"]["filters"]["limit"] + job_data["extra"]["filters"]["offset"]
+                >= job_data["extra"]["pagination"]["total_count"]
+            ):
+                break
+
+            offset += job_data["extra"]["filters"]["limit"]
+
+        return jobs
+
+    def _build_parameters(self, project_ids: List[int], environment_id: Optional[int], offset) -> dict[str, Any]:
+        parameters = {"offset": offset}
+
+        if len(project_ids) == 1:
+            parameters["project_id"] = project_ids[0]
+        elif len(project_ids) > 1:
+            project_id_str = [str(i) for i in project_ids]
+            parameters["project_id__in"] = f"[{','.join(project_id_str)}]"
+
+        if environment_id is not None:
+            parameters["environment_id"] = environment_id
+
+        logging.debug(f"Request parameters {parameters}")
+        return parameters
+
+    def _make_request(self, parameters: dict[str, Any]):
+        response = self._client.get(
+            url=f"{self.base_url}/api/v2/accounts/{self.account_id}/jobs/",
+            params=parameters,
+            headers=self._headers,
+        )
+
+        if response.status_code >= HTTP_ERROR_CODE:
+            error_data = response.json()
+            logging.error(error_data)
+            return None
+
+        return response.json()
+
+    def _check_for_creds(self):
+        """Confirm the presence of credentials"""
+        if not self._api_key:
+            raise Exception("An API key is required to get dbt Cloud jobs.")
+
+        if not self.account_id:
+            raise Exception("An account_id is required to get dbt Cloud jobs.")
+
+
+@dataclass
+class DBTCloudRefactor:
+    """Represents a single refactoring operation on a dbt Cloud job."""
+
+    rule_name: str
+    original_value: Any
+    new_value: Any
+    refactor_logs: List[str]
+
+    def to_dict(self) -> dict:
+        """Convert the refactor to a dictionary."""
+        return {
+            "rule_name": self.rule_name,
+            "original_value": self.original_value,
+            "new_value": self.new_value,
+            "refactor_logs": self.refactor_logs,
+        }
+
+
+@dataclass
+class DBTCloudChangesetResult:
+    """Represents the result of a dbt Cloud job changeset."""
+
+    dry_run: bool
+    object_type: str = "job"
+    object_id: int = 0
+    object_name: str = ""
+    original_object: Dict[str, Any] = field(default_factory=dict)
+    new_object: Dict[str, Any] = field(default_factory=dict)
+    url: str = ""
+    refactors: List[DBTCloudRefactor] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        """Convert the changeset result to a dictionary."""
+        return {
+            "dry_run": self.dry_run,
+            "object_type": self.object_type,
+            "object_id": self.object_id,
+            "object_name": self.object_name,
+            "original_object": self.original_object,
+            "new_object": self.new_object,
+            "url": self.url,
+            "refactors": [refactor.to_dict() for refactor in self.refactors],
+        }
+
+    def print_to_console(self, json_output: bool = False):
+        """Print the changeset result to the console."""
+        if not self.refactors:
+            return
+
+        if json_output:
+            print(json.dumps(self.to_dict()))
+            return
+
+        console.print(
+            f"\n{'DRY RUN - NOT APPLIED: ' if self.dry_run else ''}{self.object_type.title()} '{self.object_id} - {self.object_name}':",
+            style="green",
+        )
+        for refactor in self.refactors:
+            console.print(f"  {refactor.rule_name}", style="yellow")
+            for log in refactor.refactor_logs:
+                console.print(f"    {log}")
+
+
+def update_jobs(
+    account_id: int,
+    api_key: Optional[str],
+    base_url: str = "https://cloud.getdbt.com",
+    disable_ssl_verification: bool = False,
+    project_ids: Optional[List[int]] = None,
+    environment_ids: Optional[List[int]] = None,
+    dry_run: bool = False,
+    json_output: bool = False,
+    behavior_change: bool = False,
+):
+    """Update jobs in dbt Cloud."""
+    dbt_cloud = DBTClient(account_id, api_key, base_url, disable_ssl_verification)
+    jobs = dbt_cloud.get_jobs(project_ids, environment_ids)
+
+    changesets: List[DBTCloudChangesetResult] = []
+
+    for job in jobs:
+        modified, updated_steps = job_steps_updated(job, behavior_change)
+        execute_steps: List[str] = job.get("execute_steps", [])
+
+        if modified:
+            refactor = DBTCloudRefactor(
+                rule_name="m_selector_deprecated",
+                original_value=execute_steps,
+                new_value=updated_steps,
+                refactor_logs=[f"Updated steps from {execute_steps} to {updated_steps}"],
+            )
+
+            new_job = job.copy()
+            new_job["execute_steps"] = updated_steps
+
+            changeset = DBTCloudChangesetResult(
+                dry_run=dry_run,
+                object_id=job["id"],
+                object_name=job["name"],
+                original_object=job,
+                new_object=new_job,
+                url=f"{base_url}/deploy/{account_id}/projects/{job['project_id']}/jobs/{job['id']}/settings",
+                refactors=[refactor],
+            )
+            changesets.append(changeset)
+
+            if not dry_run:
+                dbt_cloud.update_job(new_job)
+        else:
+            changeset = DBTCloudChangesetResult(
+                dry_run=dry_run,
+                object_id=job["id"],
+                object_name=job["name"],
+                original_object=job,
+                new_object=job,
+                url=f"{base_url}/deploy/{account_id}/projects/{job['project_id']}/jobs/{job['id']}/settings",
+                refactors=[],
+            )
+            changesets.append(changeset)
+            logging.debug(f"Job '{job['id']} - {job['name']}' does not need to be updated")
+
+    # Print results
+    for changeset in changesets:
+        changeset.print_to_console(json_output)

dbt_autofix/deprecations.py

@@ -0,0 +1,20 @@
+from enum import Enum
+
+
+class DeprecationType(str, Enum):
+    """String enum for deprecation types used in DbtDeprecationRefactor."""
+
+    UNEXPECTED_JINJA_BLOCK_DEPRECATION = "UnexpectedJinjaBlockDeprecation"
+    RESOURCE_NAMES_WITH_SPACES_DEPRECATION = "ResourceNamesWithSpacesDeprecation"
+    PROPERTY_MOVED_TO_CONFIG_DEPRECATION = "PropertyMovedToConfigDeprecation"
+    CUSTOM_KEY_IN_OBJECT_DEPRECATION = "CustomKeyInObjectDeprecation"
+    MISSING_GENERIC_TEST_ARGUMENTS_PROPERTY_DEPRECATION = "MissingGenericTestArgumentsPropertyDeprecation"
+    DUPLICATE_YAML_KEYS_DEPRECATION = "DuplicateYAMLKeysDeprecation"
+    EXPOSURE_NAME_DEPRECATION = "ExposureNameDeprecation"
+    CONFIG_LOG_PATH_DEPRECATION = "ConfigLogPathDeprecation"
+    CONFIG_TARGET_PATH_DEPRECATION = "ConfigTargetPathDeprecation"
+    CONFIG_DATA_PATH_DEPRECATION = "ConfigDataPathDeprecation"
+    CONFIG_SOURCE_PATH_DEPRECATION = "ConfigSourcePathDeprecation"
+    MISSING_PLUS_PREFIX_DEPRECATION = "MissingPlusPrefixDeprecation"
+    CUSTOM_TOP_LEVEL_KEY_DEPRECATION = "CustomTopLevelKeyDeprecation"
+    CUSTOM_KEY_IN_CONFIG_DEPRECATION = "CustomKeyInConfigDeprecation"