dbt-common 1.7.0__py3-none-any.whl → 1.9.0__py3-none-any.whl

dbt_common/__about__.py

@@ -1 +1 @@
-version = "1.7.0"
+version = "1.9.0"

dbt_common/behavior_flags.py

@@ -0,0 +1,139 @@
+import inspect
+from typing import Any, Dict, List, TypedDict
+
+try:
+    from typing import NotRequired
+except ImportError:
+    # NotRequired was introduced in Python 3.11
+    # This is the suggested way to implement a TypedDict with optional arguments
+    from typing import Optional as NotRequired
+
+from dbt_common.events.functions import fire_event
+from dbt_common.events.types import BehaviorChangeEvent
+from dbt_common.exceptions import CompilationError, DbtInternalError
+
+
+class BehaviorFlag(TypedDict):
+    """
+    Configuration used to create a BehaviorFlagRendered instance
+
+    Args:
+        name: the name of the behavior flag
+        default: default setting, starts as False, becomes True after a bake-in period
+        description: an additional message to send when the flag evaluates to False
+        docs_url: the url to the relevant docs on docs.getdbt.com
+
+    *Note*:
+        While `description` and `docs_url` are both listed as `NotRequired`, at least one of them is required.
+        This is validated when the flag is rendered in `BehaviorFlagRendered` below.
+        The goal of this restriction is to provide the end user with context so they can make an informed decision
+        about if, and when, to enable the behavior flag.
+    """
+
+    name: str
+    default: bool
+    source: NotRequired[str]
+    description: NotRequired[str]
+    docs_url: NotRequired[str]
+
+
+class BehaviorFlagRendered:
+    """
+    A rendered behavior flag that gets used throughout dbt packages
+
+    Args:
+        flag: the configuration for the behavior flag
+        user_overrides: a set of user settings, one of which may be an override on this behavior flag
+    """
+
+    def __init__(self, flag: BehaviorFlag, user_overrides: Dict[str, Any]) -> None:
+        self._validate(flag)
+
+        self.name = flag["name"]
+        self.setting = user_overrides.get(flag["name"], flag["default"])
+
+        default_description = (
+            f"""The behavior controlled by `{flag["name"]}` is currently turned off.\n"""
+        )
+        default_docs_url = "https://docs.getdbt.com/reference/global-configs/behavior-changes"
+        self._behavior_change_event = BehaviorChangeEvent(
+            flag_name=flag["name"],
+            flag_source=flag.get("source", self._default_source()),
+            description=flag.get("description", default_description),
+            docs_url=flag.get("docs_url", default_docs_url),
+        )
+
+    @staticmethod
+    def _validate(flag: BehaviorFlag) -> None:
+        if flag.get("description") is None and flag.get("docs_url") is None:
+            raise DbtInternalError(
+                "Behavior change flags require at least one of `description` and `docs_url`."
+            )
+
+    @property
+    def setting(self) -> bool:
+        if self._setting is False:
+            fire_event(self._behavior_change_event)
+        return self._setting
+
+    @setting.setter
+    def setting(self, value: bool) -> None:
+        self._setting = value
+
+    @property
+    def no_warn(self) -> bool:
+        return self._setting
+
+    @staticmethod
+    def _default_source() -> str:
+        """
+        If the maintainer did not provide a source, default to the module that called this class.
+        For adapters, this will likely be `dbt.adapters.<foo>.impl` for `dbt-foo`.
+        """
+        for frame in inspect.stack():
+            if module := inspect.getmodule(frame[0]):
+                if module.__name__ != __name__:
+                    return module.__name__
+        return "Unknown"
+
+    def __bool__(self) -> bool:
+        return self.setting
+
+
+class Behavior:
+    """
+    A collection of behavior flags
+
+    This is effectively a dictionary that supports dot notation for easy reference, e.g.:
+        ```python
+        if adapter.behavior.my_flag:
+            ...
+
+        if adapter.behavior.my_flag.no_warn:  # this will not fire the behavior change event
+            ...
+        ```
+        ```jinja
+        {% if adapter.behavior.my_flag %}
+            ...
+        {% endif %}
+
+        {% if adapter.behavior.my_flag.no_warn %}  {# this will not fire the behavior change event #}
+            ...
+        {% endif %}
+        ```
+
+    Args:
+        flags: a list of configurations, one for each behavior flag
+        user_overrides: a set of user settings, which may include overrides on one or more of the behavior flags
+    """
+
+    _flags: List[BehaviorFlagRendered]
+
+    def __init__(self, flags: List[BehaviorFlag], user_overrides: Dict[str, Any]) -> None:
+        self._flags = [BehaviorFlagRendered(flag, user_overrides) for flag in flags]
+
+    def __getattr__(self, name: str) -> BehaviorFlagRendered:
+        for flag in self._flags:
+            if flag.name == name:
+                return flag
+        raise CompilationError(f"The flag {name} has not be registered.")

dbt_common/clients/agate_helper.py

@@ -1,6 +1,6 @@
 from codecs import BOM_UTF8
 
-import agate  # type: ignore
+import agate
 import datetime
 import isodate
 import json
@@ -149,7 +149,7 @@ def as_matrix(table):
     return [r.values() for r in table.rows.values()]
 
 
-def from_csv(abspath, text_columns, delimiter=","):
+def from_csv(abspath, text_columns, delimiter=",") -> agate.Table:
     type_tester = build_type_tester(text_columns=text_columns)
     with open(abspath, encoding="utf-8") as fp:
         if fp.read(1) != BOM:

dbt_common/clients/jinja.py

@@ -6,15 +6,29 @@ from ast import literal_eval
 from collections import ChainMap
 from contextlib import contextmanager
 from itertools import chain, islice
-from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Union, Set, Type
+from types import CodeType
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    Mapping,
+    Optional,
+    Union,
+    Set,
+    Type,
+    NoReturn,
+)
+
 from typing_extensions import Protocol
 
-import jinja2  # type: ignore
-import jinja2.ext  # type: ignore
-import jinja2.nativetypes  # type: ignore
-import jinja2.nodes  # type: ignore
-import jinja2.parser  # type: ignore
-import jinja2.sandbox  # type: ignore
+import jinja2
+import jinja2.ext
+import jinja2.nativetypes
+import jinja2.nodes
+import jinja2.parser
+import jinja2.sandbox
 
 from dbt_common.tests import test_caching_enabled
 from dbt_common.utils.jinja import (
@@ -39,10 +53,17 @@ from dbt_common.exceptions.macros import MacroReturn, UndefinedMacroError, Caugh
 SUPPORTED_LANG_ARG = jinja2.nodes.Name("supported_languages", "param")
 
 # Global which can be set by dependents of dbt-common (e.g. core via flag parsing)
-MACRO_DEBUGGING = False
+MACRO_DEBUGGING: Union[str, bool] = False
+
+_ParseReturn = Union[jinja2.nodes.Node, List[jinja2.nodes.Node]]
+
+
+# Temporary type capturing the concept the functions in this file expect for a "node"
+class _NodeProtocol(Protocol):
+    pass
 
 
-def _linecache_inject(source, write):
+def _linecache_inject(source: str, write: bool) -> str:
     if write:
         # this is the only reliable way to accomplish this. Obviously, it's
         # really darn noisy and will fill your temporary directory
@@ -58,18 +79,18 @@ def _linecache_inject(source, write):
     else:
         # `codecs.encode` actually takes a `bytes` as the first argument if
         # the second argument is 'hex' - mypy does not know this.
-        rnd = codecs.encode(os.urandom(12), "hex")  # type: ignore
+        rnd = codecs.encode(os.urandom(12), "hex")
         filename = rnd.decode("ascii")
 
     # put ourselves in the cache
     cache_entry = (len(source), None, [line + "\n" for line in source.splitlines()], filename)
     # linecache does in fact have an attribute `cache`, thanks
-    linecache.cache[filename] = cache_entry  # type: ignore
+    linecache.cache[filename] = cache_entry
     return filename
 
 
 class MacroFuzzParser(jinja2.parser.Parser):
-    def parse_macro(self):
+    def parse_macro(self) -> jinja2.nodes.Macro:
         node = jinja2.nodes.Macro(lineno=next(self.stream).lineno)
 
         # modified to fuzz macros defined in the same file. this way
@@ -83,16 +104,13 @@ class MacroFuzzParser(jinja2.parser.Parser):
 
 
 class MacroFuzzEnvironment(jinja2.sandbox.SandboxedEnvironment):
-    def _parse(self, source, name, filename):
+    def _parse(
+        self, source: str, name: Optional[str], filename: Optional[str]
+    ) -> jinja2.nodes.Template:
         return MacroFuzzParser(self, source, name, filename).parse()
 
-    def _compile(self, source, filename):
+    def _compile(self, source: str, filename: str) -> CodeType:
         """
-
-
-
-
-
         Override jinja's compilation. Use to stash the rendered source inside
         the python linecache for debugging when the appropriate environment
         variable is set.
@@ -108,7 +126,7 @@ class MacroFuzzEnvironment(jinja2.sandbox.SandboxedEnvironment):
 
 
 class MacroFuzzTemplate(jinja2.nativetypes.NativeTemplate):
-    environment_class = MacroFuzzEnvironment
+    environment_class = MacroFuzzEnvironment  # type: ignore
 
     def new_context(
         self,
@@ -124,6 +142,7 @@ class MacroFuzzTemplate(jinja2.nativetypes.NativeTemplate):
                 "shared or locals parameters."
             )
 
+        vars = {} if vars is None else vars
         parent = ChainMap(vars, self.globals) if self.globals else vars
 
         return self.environment.context_class(self.environment, parent, self.name, self.blocks)
@@ -170,11 +189,11 @@ class NumberMarker(NativeMarker):
     pass
 
 
-def _is_number(value) -> bool:
+def _is_number(value: Any) -> bool:
     return isinstance(value, (int, float)) and not isinstance(value, bool)
 
 
-def quoted_native_concat(nodes):
+def quoted_native_concat(nodes: Iterator[str]) -> Any:
     """Handle special case for native_concat from the NativeTemplate.
 
     This is almost native_concat from the NativeTemplate, except in the
@@ -212,7 +231,7 @@ def quoted_native_concat(nodes):
 class NativeSandboxTemplate(jinja2.nativetypes.NativeTemplate):  # mypy: ignore
     environment_class = NativeSandboxEnvironment  # type: ignore
 
-    def render(self, *args, **kwargs):
+    def render(self, *args: Any, **kwargs: Any) -> Any:
         """Render the template to produce a native Python type.
 
         If the result is a single node, its value is returned. Otherwise,
@@ -228,6 +247,11 @@ class NativeSandboxTemplate(jinja2.nativetypes.NativeTemplate):  # mypy: ignore
             return self.environment.handle_exception()
 
 
+class MacroProtocol(Protocol):
+    name: str
+    macro_sql: str
+
+
 NativeSandboxEnvironment.template_class = NativeSandboxTemplate  # type: ignore
 
 
@@ -235,7 +259,7 @@ class TemplateCache:
     def __init__(self) -> None:
         self.file_cache: Dict[str, jinja2.Template] = {}
 
-    def get_node_template(self, node) -> jinja2.Template:
+    def get_node_template(self, node: MacroProtocol) -> jinja2.Template:
         key = node.macro_sql
 
         if key in self.file_cache:
@@ -250,7 +274,7 @@ class TemplateCache:
         self.file_cache[key] = template
         return template
 
-    def clear(self):
+    def clear(self) -> None:
         self.file_cache.clear()
 
 
@@ -261,13 +285,13 @@ class BaseMacroGenerator:
     def __init__(self, context: Optional[Dict[str, Any]] = None) -> None:
         self.context: Optional[Dict[str, Any]] = context
 
-    def get_template(self):
+    def get_template(self) -> jinja2.Template:
         raise NotImplementedError("get_template not implemented!")
 
     def get_name(self) -> str:
         raise NotImplementedError("get_name not implemented!")
 
-    def get_macro(self):
+    def get_macro(self) -> Callable:
         name = self.get_name()
         template = self.get_template()
         # make the module. previously we set both vars and local, but that's
@@ -285,7 +309,7 @@ class BaseMacroGenerator:
         except (TypeError, jinja2.exceptions.TemplateRuntimeError) as e:
             raise CaughtMacroError(e)
 
-    def call_macro(self, *args, **kwargs):
+    def call_macro(self, *args: Any, **kwargs: Any) -> Any:
         # called from __call__ methods
         if self.context is None:
             raise DbtInternalError("Context is still None in call_macro!")
@@ -300,11 +324,6 @@ class BaseMacroGenerator:
                 return e.value
 
 
-class MacroProtocol(Protocol):
-    name: str
-    macro_sql: str
-
-
 class CallableMacroGenerator(BaseMacroGenerator):
     def __init__(
         self,
@@ -314,7 +333,7 @@ class CallableMacroGenerator(BaseMacroGenerator):
         super().__init__(context)
         self.macro = macro
 
-    def get_template(self):
+    def get_template(self) -> jinja2.Template:
         return template_cache.get_node_template(self.macro)
 
     def get_name(self) -> str:
@@ -331,14 +350,14 @@ class CallableMacroGenerator(BaseMacroGenerator):
             raise e
 
     # this makes MacroGenerator objects callable like functions
-    def __call__(self, *args, **kwargs):
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
         return self.call_macro(*args, **kwargs)
 
 
 class MaterializationExtension(jinja2.ext.Extension):
     tags = ["materialization"]
 
-    def parse(self, parser):
+    def parse(self, parser: jinja2.parser.Parser) -> _ParseReturn:
         node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
         materialization_name = parser.parse_assign_target(name_only=True).name
 
@@ -381,7 +400,7 @@ class MaterializationExtension(jinja2.ext.Extension):
 class DocumentationExtension(jinja2.ext.Extension):
     tags = ["docs"]
 
-    def parse(self, parser):
+    def parse(self, parser: jinja2.parser.Parser) -> _ParseReturn:
         node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
         docs_name = parser.parse_assign_target(name_only=True).name
 
@@ -395,7 +414,7 @@ class DocumentationExtension(jinja2.ext.Extension):
 class TestExtension(jinja2.ext.Extension):
     tags = ["test"]
 
-    def parse(self, parser):
+    def parse(self, parser: jinja2.parser.Parser) -> _ParseReturn:
         node = jinja2.nodes.Macro(lineno=next(parser.stream).lineno)
         test_name = parser.parse_assign_target(name_only=True).name
 
@@ -405,13 +424,19 @@ class TestExtension(jinja2.ext.Extension):
         return node
 
 
-def _is_dunder_name(name):
+def _is_dunder_name(name: str) -> bool:
     return name.startswith("__") and name.endswith("__")
 
 
-def create_undefined(node=None):
+def create_undefined(node: Optional[_NodeProtocol] = None) -> Type[jinja2.Undefined]:
     class Undefined(jinja2.Undefined):
-        def __init__(self, hint=None, obj=None, name=None, exc=None):
+        def __init__(
+            self,
+            hint: Optional[str] = None,
+            obj: Any = None,
+            name: Optional[str] = None,
+            exc: Any = None,
+        ) -> None:
             super().__init__(hint=hint, name=name)
             self.node = node
             self.name = name
@@ -421,12 +446,12 @@ def create_undefined(node=None):
             self.unsafe_callable = False
             self.alters_data = False
 
-        def __getitem__(self, name):
+        def __getitem__(self, name: Any) -> "Undefined":
             # Propagate the undefined value if a caller accesses this as if it
             # were a dictionary
             return self
 
-        def __getattr__(self, name):
+        def __getattr__(self, name: str) -> "Undefined":
             if name == "name" or _is_dunder_name(name):
                 raise AttributeError(
                     "'{}' object has no attribute '{}'".format(type(self).__name__, name)
@@ -436,11 +461,11 @@ def create_undefined(node=None):
 
             return self.__class__(hint=self.hint, name=self.name)
 
-        def __call__(self, *args, **kwargs):
+        def __call__(self, *args: Any, **kwargs: Any) -> "Undefined":
             return self
 
-        def __reduce__(self):
-            raise UndefinedCompilationError(name=self.name, node=node)
+        def __reduce__(self) -> NoReturn:
+            raise UndefinedCompilationError(name=self.name or "unknown", node=node)
 
     return Undefined
 
@@ -462,7 +487,7 @@ TEXT_FILTERS: Dict[str, Callable[[Any], Any]] = {
 
 
 def get_environment(
-    node=None,
+    node: Optional[_NodeProtocol] = None,
     capture_macros: bool = False,
     native: bool = False,
 ) -> jinja2.Environment:
@@ -471,7 +496,7 @@ def get_environment(
     }
 
     if capture_macros:
-        args["undefined"] = create_undefined(node)
+        args["undefined"] = create_undefined(node)  # type: ignore
 
     args["extensions"].append(MaterializationExtension)
     args["extensions"].append(DocumentationExtension)
@@ -492,7 +517,7 @@ def get_environment(
 
 
 @contextmanager
-def catch_jinja(node=None) -> Iterator[None]:
+def catch_jinja(node: Optional[_NodeProtocol] = None) -> Iterator[None]:
     try:
         yield
     except jinja2.exceptions.TemplateSyntaxError as e:
@@ -505,16 +530,16 @@ def catch_jinja(node=None) -> Iterator[None]:
         raise
 
 
-_TESTING_PARSE_CACHE: Dict[str, jinja2.Template] = {}
+_TESTING_PARSE_CACHE: Dict[str, jinja2.nodes.Template] = {}
 
 
-def parse(string):
+def parse(string: Any) -> jinja2.nodes.Template:
     str_string = str(string)
     if test_caching_enabled() and str_string in _TESTING_PARSE_CACHE:
         return _TESTING_PARSE_CACHE[str_string]
 
     with catch_jinja():
-        parsed = get_environment().parse(str(string))
+        parsed: jinja2.nodes.Template = get_environment().parse(str(string))
         if test_caching_enabled():
             _TESTING_PARSE_CACHE[str_string] = parsed
         return parsed
@@ -523,10 +548,10 @@ def parse(string):
 def get_template(
     string: str,
     ctx: Dict[str, Any],
-    node=None,
+    node: Optional[_NodeProtocol] = None,
     capture_macros: bool = False,
     native: bool = False,
-):
+) -> jinja2.Template:
     with catch_jinja(node):
         env = get_environment(node, capture_macros, native=native)
 
@@ -534,7 +559,9 @@ def get_template(
         return env.from_string(template_source, globals=ctx)
 
 
-def render_template(template, ctx: Dict[str, Any], node=None) -> str:
+def render_template(
+    template: jinja2.Template, ctx: Dict[str, Any], node: Optional[_NodeProtocol] = None
+) -> str:
     with catch_jinja(node):
         return template.render(ctx)
 
@@ -544,6 +571,7 @@ _TESTING_BLOCKS_CACHE: Dict[int, List[Union[BlockData, BlockTag]]] = {}
 
 def _get_blocks_hash(text: str, allowed_blocks: Optional[Set[str]], collect_raw_data: bool) -> int:
     """Provides a hash function over the arguments to extract_toplevel_blocks, in order to support caching."""
+    allowed_blocks = allowed_blocks or set()
     allowed_tuple = tuple(sorted(allowed_blocks) or [])
     return text.__hash__() + allowed_tuple.__hash__() + collect_raw_data.__hash__()
 

dbt_common/clients/system.py

@@ -52,6 +52,7 @@ class FindMatchingParams:
     root_path: str
     relative_paths_to_search: List[str]
     file_pattern: str
+
     # ignore_spec: Optional[PathSpec] = None
 
     def __init__(
@@ -608,11 +609,36 @@ def rename(from_path: str, to_path: str, force: bool = False) -> None:
     shutil.move(from_path, to_path)
 
 
+def safe_extract(tarball: tarfile.TarFile, path: str = ".") -> None:
+    """
+    Fix for CWE-22: Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal')
+    Solution copied from https://github.com/mindsdb/mindsdb/blob/main/mindsdb/utilities/fs.py
+    """
+
+    def _is_within_directory(directory, target):
+        abs_directory = os.path.abspath(directory)
+        abs_target = os.path.abspath(target)
+        prefix = os.path.commonprefix([abs_directory, abs_target])
+        return prefix == abs_directory
+
+    # for py >= 3.12
+    if hasattr(tarball, "data_filter"):
+        tarball.extractall(path, filter="data")
+    else:
+        members = tarball.getmembers()
+        for member in members:
+            member_path = os.path.join(path, member.name)
+            if not _is_within_directory(path, member_path):
+                raise tarfile.OutsideDestinationError(member, path)
+
+        tarball.extractall(path, members=members)
+
+
 def untar_package(tar_path: str, dest_dir: str, rename_to: Optional[str] = None) -> None:
     tar_path = convert_path(tar_path)
     tar_dir_name = None
     with tarfile.open(tar_path, "r:gz") as tarball:
-        tarball.extractall(dest_dir)
+        safe_extract(tarball, dest_dir)
         tar_dir_name = os.path.commonprefix(tarball.getnames())
     if rename_to:
         downloaded_path = os.path.join(dest_dir, tar_dir_name)

dbt_common/context.py

@@ -6,7 +6,7 @@ from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX
 from dbt_common.record import Recorder
 
 
-class CaseInsensitiveMapping(Mapping):
+class CaseInsensitiveMapping(Mapping[str, str]):
     def __init__(self, env: Mapping[str, str]):
         self._env = {k.casefold(): (k, v) for k, v in env.items()}
 
@@ -65,7 +65,7 @@ _INVOCATION_CONTEXT_VAR: ContextVar[InvocationContext] = ContextVar("DBT_INVOCAT
 
 
 def reliably_get_invocation_var() -> ContextVar[InvocationContext]:
-    invocation_var: Optional[ContextVar] = next(
+    invocation_var: Optional[ContextVar[InvocationContext]] = next(
         (cv for cv in copy_context() if cv.name == _INVOCATION_CONTEXT_VAR.name), None
     )