dbt-common 1.15.0__py3-none-any.whl → 1.17.0__py3-none-any.whl

dbt_common/__about__.py

@@ -1 +1 @@
-version = "1.15.0"
+version = "1.17.0"

dbt_common/context.py

@@ -43,6 +43,12 @@ class InvocationContext:
         self._env_secrets: Optional[List[str]] = None
         self._env_private = env_private
         self.recorder: Optional[Recorder] = None
+
+        # If set to True later, this flag will prevent dbt from creating a new
+        # invocation context for every invocation, which is useful for testing
+        # scenarios.
+        self.do_not_reset = False
+
         # This class will also eventually manage the invocation_id, flags, event manager, etc.
 
     @property
@@ -85,3 +91,10 @@ def get_invocation_context() -> InvocationContext:
     invocation_var = reliably_get_invocation_var()
     ctx = invocation_var.get()
     return ctx
+
+
+def try_get_invocation_context() -> Optional[InvocationContext]:
+    try:
+        return get_invocation_context()
+    except Exception:
+        return None

dbt_common/events/event_manager.py

@@ -1,19 +1,66 @@
 import os
 import traceback
-from typing import List, Optional, Protocol, Tuple
+from typing import Any, List, Optional, Protocol, Tuple
 
 from dbt_common.events.base_types import BaseEvent, EventLevel, msg_from_base_event, TCallback
 from dbt_common.events.logger import LoggerConfig, _Logger, _TextLogger, _JsonLogger, LineFormat
+from dbt_common.exceptions.events import EventCompilationError
+from dbt_common.helper_types import WarnErrorOptions
 
 
 class EventManager:
     def __init__(self) -> None:
         self.loggers: List[_Logger] = []
         self.callbacks: List[TCallback] = []
-
-    def fire_event(self, e: BaseEvent, level: Optional[EventLevel] = None) -> None:
+        self._warn_error: Optional[bool] = None
+        self._warn_error_options: Optional[WarnErrorOptions] = None
+        self.require_warn_or_error_handling: bool = False
+
+    @property
+    def warn_error(self) -> bool:
+        if self._warn_error is None:
+            from dbt_common.events.functions import WARN_ERROR
+
+            return WARN_ERROR
+        return self._warn_error
+
+    @warn_error.setter
+    def warn_error(self, warn_error: bool) -> None:
+        self._warn_error = warn_error
+
+    @property
+    def warn_error_options(self) -> WarnErrorOptions:
+        if self._warn_error_options is None:
+            from dbt_common.events.functions import WARN_ERROR_OPTIONS
+
+            return WARN_ERROR_OPTIONS
+        return self._warn_error_options
+
+    @warn_error_options.setter
+    def warn_error_options(self, warn_error_options: WarnErrorOptions) -> None:
+        self._warn_error_options = warn_error_options
+
+    def fire_event(
+        self,
+        e: BaseEvent,
+        level: Optional[EventLevel] = None,
+        node: Any = None,
+        force_warn_or_error_handling: bool = False,
+    ) -> None:
         msg = msg_from_base_event(e, level=level)
 
+        if force_warn_or_error_handling or (
+            self.require_warn_or_error_handling and msg.info.level == "warn"
+        ):
+            event_name = type(e).__name__
+            if self.warn_error or self.warn_error_options.includes(event_name):
+                # This has the potential to create an infinite loop if the handling of the raised
+                # EventCompilationError fires an event as a warning instead of an error.
+                raise EventCompilationError(e.message(), node)
+            elif self.warn_error_options.silenced(event_name):
+                # Return early if the event is silenced
+                return
+
         if os.environ.get("DBT_TEST_BINARY_SERIALIZATION"):
             print(f"--- {msg.info.name}")
             try:
@@ -48,8 +95,17 @@ class EventManager:
 class IEventManager(Protocol):
     callbacks: List[TCallback]
     loggers: List[_Logger]
-
-    def fire_event(self, e: BaseEvent, level: Optional[EventLevel] = None) -> None:
+    warn_error: bool
+    warn_error_options: WarnErrorOptions
+    require_warn_or_error_handling: bool
+
+    def fire_event(
+        self,
+        e: BaseEvent,
+        level: Optional[EventLevel] = None,
+        node: Any = None,
+        force_warn_or_error_handling: bool = False,
+    ) -> None:
         ...
 
     def add_logger(self, config: LoggerConfig) -> None:
@@ -65,8 +121,17 @@ class TestEventManager(IEventManager):
     def __init__(self) -> None:
         self.event_history: List[Tuple[BaseEvent, Optional[EventLevel]]] = []
         self.loggers = []
-
-    def fire_event(self, e: BaseEvent, level: Optional[EventLevel] = None) -> None:
+        self.warn_error = False
+        self.warn_error_options = WarnErrorOptions(include=[], exclude=[])
+        self.require_warn_or_error_handling = False
+
+    def fire_event(
+        self,
+        e: BaseEvent,
+        level: Optional[EventLevel] = None,
+        node: Any = None,
+        force_warn_or_error_handling: bool = False,
+    ) -> None:
         self.event_history.append((e, level))
 
     def add_logger(self, config: LoggerConfig) -> None:

dbt_common/events/functions.py

@@ -1,9 +1,8 @@
 from pathlib import Path
 
 from dbt_common.events.event_manager_client import get_event_manager
-from dbt_common.exceptions import EventCompilationError
-from dbt_common.invocation import get_invocation_id
 from dbt_common.helper_types import WarnErrorOptions
+from dbt_common.invocation import get_invocation_id
 from dbt_common.utils.encoding import ForgivingJSONEncoder
 from dbt_common.events.base_types import BaseEvent, EventLevel, EventMsg
 from dbt_common.events.logger import LoggerConfig, LineFormat
@@ -13,7 +12,7 @@ from functools import partial
 import json
 import os
 import sys
-from typing import Callable, Dict, Optional, TextIO, Union
+from typing import Any, Callable, Dict, Optional, TextIO, Union
 from google.protobuf.json_format import MessageToDict
 
 LOG_VERSION = 3
@@ -114,12 +113,9 @@ def msg_to_dict(msg: EventMsg) -> dict:
     return msg_dict
 
 
+# This function continues to exist to provide backwards compatibility
 def warn_or_error(event, node=None) -> None:
-    event_name = type(event).__name__
-    if WARN_ERROR or WARN_ERROR_OPTIONS.includes(event_name):
-        raise EventCompilationError(event.message(), node)
-    elif not WARN_ERROR_OPTIONS.silenced(event_name):
-        fire_event(event)
+    fire_event(e=event, node=node, force_warn_or_error_handling=True)
 
 
 # an alternative to fire_event which only creates and logs the event value
@@ -135,8 +131,15 @@ def fire_event_if(
 # this is where all the side effects happen branched by event type
 # (i.e. - mutating the event history, printing to stdout, logging
 # to files, etc.)
-def fire_event(e: BaseEvent, level: Optional[EventLevel] = None) -> None:
-    get_event_manager().fire_event(e, level=level)
+def fire_event(
+    e: BaseEvent,
+    level: Optional[EventLevel] = None,
+    node: Any = None,
+    force_warn_or_error_handling: bool = False,
+) -> None:
+    get_event_manager().fire_event(
+        e, level=level, node=node, force_warn_or_error_handling=force_warn_or_error_handling
+    )
 
 
 def get_metadata_vars() -> Dict[str, str]:

dbt_common/record.py

@@ -12,13 +12,16 @@ import json
 import os
 
 from enum import Enum
+from threading import Lock
 from typing import Any, Callable, Dict, List, Mapping, Optional, TextIO, Tuple, Type
-import contextvars
 
 from mashumaro import field_options
 from mashumaro.mixins.json import DataClassJSONMixin
 from mashumaro.types import SerializationStrategy
 
+import contextvars
+
+
 RECORDED_BY_HIGHER_FUNCTION = contextvars.ContextVar("RECORDED_BY_HIGHER_FUNCTION", default=False)
 
 
@@ -31,9 +34,10 @@ class Record:
     result_cls: Optional[type] = None
     group: Optional[str] = None
 
-    def __init__(self, params, result) -> None:
+    def __init__(self, params, result, seq=None) -> None:
         self.params = params
         self.result = result
+        self.seq = seq
 
     def to_dict(self) -> Dict[str, Any]:
         return {
@@ -45,6 +49,7 @@ class Record:
             else dataclasses.asdict(self.result)
             if self.result is not None
             else None,
+            "seq": self.seq,
         }
 
     @classmethod
@@ -61,7 +66,8 @@ class Record:
             if cls.result_cls is not None
             else None
         )
-        return cls(params=p, result=r)
+        s = dct.get("seq", None)
+        return cls(params=p, result=r, seq=s)
 
 
 class Diff:
@@ -145,6 +151,7 @@ class Recorder:
         types: Optional[List],
         current_recording_path: str = "recording.json",
         previous_recording_path: Optional[str] = None,
+        in_memory: bool = False,
     ) -> None:
         self.mode = mode
         self.recorded_types = types
@@ -167,6 +174,18 @@ class Recorder:
             if self.mode == RecorderMode.REPLAY:
                 self._unprocessed_records_by_type = self.load(self.previous_recording_path)
 
+        self._counter = 0
+        self._counter_lock = Lock()
+
+        self._record_added = False
+        self._recording_file: Optional[TextIO] = None
+        if mode == RecorderMode.RECORD and not in_memory:
+            self._recording_file = open(current_recording_path, "w")
+            self._recording_file.write("[")
+
+    def __del__(self):
+        self.clean_up_stream()
+
     @classmethod
     def register_record_type(cls, rec_type) -> Any:
         cls._record_cls_by_name[rec_type.__name__] = rec_type
@@ -175,9 +194,26 @@ class Recorder:
 
     def add_record(self, record: Record) -> None:
         rec_cls_name = record.__class__.__name__  # type: ignore
-        if rec_cls_name not in self._records_by_type:
-            self._records_by_type[rec_cls_name] = []
-        self._records_by_type[rec_cls_name].append(record)
+
+        with self._counter_lock:
+            record.seq = self._counter
+            self._counter += 1
+
+        if self._recording_file is not None:
+            if self._record_added:
+                self._recording_file.write(",")
+            try:
+                dct = Recorder._get_tagged_dict(record, rec_cls_name)
+                json.dump(dct, self._recording_file)
+                self._record_added = True
+            except Exception:
+                json.dump(
+                    {"type": "RecordingError", "record_type": rec_cls_name}, self._recording_file
+                )
+        else:
+            if rec_cls_name not in self._records_by_type:
+                self._records_by_type[rec_cls_name] = []
+            self._records_by_type[rec_cls_name].append(record)
 
     def pop_matching_record(self, params: Any) -> Optional[Record]:
         rec_type_name = self._record_name_by_params_name.get(type(params).__name__)
@@ -199,21 +235,39 @@ class Recorder:
         return match
 
     def write_json(self, out_stream: TextIO):
-        d = self._to_dict()
+        d = self._to_list()
         json.dump(d, out_stream)
 
     def write(self) -> None:
-        with open(self.current_recording_path, "w") as file:
-            self.write_json(file)
-
-    def _to_dict(self) -> Dict:
-        dct: Dict[str, Any] = {}
-
+        if self._recording_file is not None:
+            self.clean_up_stream()
+        else:
+            with open(self.current_recording_path, "w") as file:
+                self.write_json(file)
+
+    def clean_up_stream(self) -> None:
+        if self._recording_file is not None:
+            self._recording_file.write("]")
+            self._recording_file.close()
+            self._recording_file = None
+
+    @staticmethod
+    def _get_tagged_dict(record: Record, record_type: str) -> Dict:
+        d = record.to_dict()
+        d["type"] = record_type
+        return d
+
+    def _to_list(self) -> List[Dict]:
+        record_list: List[Dict] = []
         for record_type in self._records_by_type:
-            record_list = [r.to_dict() for r in self._records_by_type[record_type]]
-            dct[record_type] = record_list
+            record_list.extend(
+                Recorder._get_tagged_dict(r, record_type)
+                for r in self._records_by_type[record_type]
+            )
+
+        record_list.sort(key=lambda r: r["seq"])
 
-        return dct
+        return record_list
 
     @classmethod
     def load(cls, file_name: str) -> Dict[str, List[Dict[str, Any]]]:
@@ -331,7 +385,6 @@ def auto_record_function(
         None,
         group,
         index_on_thread_name,
-        False,
     )
 
 
@@ -340,6 +393,7 @@ def record_function(
     method: bool = False,
     tuple_result: bool = False,
     id_field_name: Optional[str] = None,
+    index_on_thread_id: bool = False,
 ) -> Callable:
     """This is the @record_function decorator, which marks functions which will
     have their function calls recorded during record mode, and mocked out with
@@ -351,8 +405,7 @@ def record_function(
         tuple_result,
         id_field_name,
         None,
-        False,
-        False,
+        index_on_thread_id,
     )
 
 
@@ -402,7 +455,6 @@ def _record_function_inner(
     id_field_name,
     group,
     index_on_thread_id,
-    is_classmethod,
     func_to_record,
 ):
     if isinstance(record_type, str):
@@ -442,7 +494,7 @@ def _record_function_inner(
         except LookupError:
             pass
 
-        call_args = args[1:] if is_classmethod else args
+        call_args = args
 
         if recorder is None:
             return func_to_record(*call_args, **kwargs)
@@ -458,9 +510,16 @@ def _record_function_inner(
         param_args = args[1:] if method else args
         if method and id_field_name is not None:
             if index_on_thread_id:
-                from dbt_common.context import get_invocation_context
+                from dbt_common.events.contextvars import get_node_info
+
+                node_info = get_node_info()
+                if node_info and "unique_id" in node_info:
+                    thread_name = node_info["unique_id"]
+                else:
+                    from dbt_common.context import get_invocation_context
 
-                param_args = (get_invocation_context().name,) + param_args
+                    thread_name = get_invocation_context().name
+                param_args = (thread_name,) + param_args
             else:
                 param_args = (getattr(args[0], id_field_name),) + param_args
 
@@ -532,21 +591,25 @@ def supports_replay(cls):
             metadata = getattr(method, "_record_metadata", None)
             if method and getattr(method, "_record_metadata", None):
                 sub_method = getattr(sub_cls, method_name, None)
-                recorded_sub_method = _record_function_inner(
-                    metadata["record_type"],
-                    metadata["method"],
-                    metadata["tuple_result"],
-                    metadata["id_field_name"],
-                    metadata["group"],
-                    metadata["index_on_thread_id"],
-                    _is_classmethod(method),
-                    sub_method,
-                )
+                sub_method_metadata = getattr(sub_method, "_record_metadata", None)
+
+                if not sub_method_metadata:
+                    recorded_sub_method = _record_function_inner(
+                        metadata["record_type"],
+                        metadata["method"],
+                        metadata["tuple_result"],
+                        metadata["id_field_name"],
+                        metadata["group"],
+                        metadata["index_on_thread_id"],
+                        sub_method.__func__
+                        if _is_classmethod(method)
+                        else sub_method,  # unwrap if classmethod
+                    )
 
-                if _is_classmethod(method):
-                    recorded_sub_method = classmethod(recorded_sub_method)
+                    if _is_classmethod(method):
+                        # rewrap if submethod
+                        recorded_sub_method = classmethod(recorded_sub_method)
 
-                if sub_method is not None:
                     setattr(
                         sub_cls,
                         method_name,

{dbt_common-1.15.0.dist-info → dbt_common-1.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbt-common
-Version: 1.15.0
+Version: 1.17.0
 Summary: The shared common utilities that dbt-core and adapter implementations use
 Project-URL: Homepage, https://github.com/dbt-labs/dbt-common
 Project-URL: Repository, https://github.com/dbt-labs/dbt-common.git

{dbt_common-1.15.0.dist-info → dbt_common-1.17.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
-dbt_common/__about__.py,sha256=ifS0ZWsb82NyM4xyQWKPKGKE5bG3EXlYHT1H7h1HxOY,19
+dbt_common/__about__.py,sha256=uYViEABuLbLykJW0jxwSXYZEExCXp3lcdeO8uJWFORw,19
 dbt_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbt_common/behavior_flags.py,sha256=hQzxCqQSweJbRp_xoQqNnlUF77PBuOdCdLOSdcBlkxk,4885
 dbt_common/constants.py,sha256=-Y5DIL1SDPQWtlCNizXRYxFgbx1D7LaLs1ysamvGMRk,278
-dbt_common/context.py,sha256=tVeXtsptvuw7d8CvdlYSBFcKLyLZ852iQNwcxNmUzYY,2577
+dbt_common/context.py,sha256=-ErtKG4xfOh1-Y569fwu6u2O381nRan18HhATrYDoZE,2950
 dbt_common/dataclass_schema.py,sha256=u2S0dxwxIghv8RMqC91HlWZJVxmsC_844yZQaGyOwdY,5563
 dbt_common/helper_types.py,sha256=FWJGPmp7Qp2iToHyI4uvhkBbu_d1tl2_oF-obi98_N4,3917
 dbt_common/invocation.py,sha256=xw0NBIE-6LHd135cx4non-MkGGsia0mYN0lMkmNEucE,435
 dbt_common/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dbt_common/record.py,sha256=ExF8ccUpDv2kButAMnTR8zSyD9agU7nhXTKFDx7Iw7Y,19352
+dbt_common/record.py,sha256=90NwLo0jZQkdQwS3QMnvEmtVrBESvIhP-1LMfXfE3rA,21620
 dbt_common/semver.py,sha256=Znewz6tc_NBpXr4mZf20bK_RayPL4ODrnxDbkUZrrRo,15034
 dbt_common/tests.py,sha256=6lC_JuRtoYO6cbAF8-R5aTM4HtQiM_EH8X5m_97duGY,315
 dbt_common/ui.py,sha256=rc2TEM29raBFc_LXcg901pMDD07C2ohwp9qzkE-7pBY,2567
@@ -30,10 +30,10 @@ dbt_common/events/__init__.py,sha256=av08vfpxo0ek7PqZNtMxY8FODJ3xwph4ehRxgInx4LA
 dbt_common/events/base_types.py,sha256=bdDMbawAV0FkxmvuxgsTev82vxTuyu6rJiSkvEQPsO8,5525
 dbt_common/events/contextvars.py,sha256=EIs1P6NrJzx_IAV17x5cVqOAS4Lqbu6oc0etHtWCJOo,3097
 dbt_common/events/event_handler.py,sha256=jfi0PyqIOGnXCG9HEa0VIVULqNvXs1RYmAg0b50ChQs,1385
-dbt_common/events/event_manager.py,sha256=IIUwSyt_RcBbUI_iE5mnpmZt2uW7lG49RXOWz2VlUv0,2300
+dbt_common/events/event_manager.py,sha256=0CDHNh_qTjdj4blpJiJ7gq6YkP1NzjE0iIct4v_MdoA,4588
 dbt_common/events/event_manager_client.py,sha256=VKlIYJPcexmDKnidkyrs8BIuNZ1_CwDFGz-gBM2SAvo,1193
 dbt_common/events/format.py,sha256=x1RWDZ8G7ZMHmxdld6Q4VXca4kvnhiQOIaQXkC6Uo0Q,1609
-dbt_common/events/functions.py,sha256=_7CLApCKb9KhurOgfVRpW-yGKGE_yjYUguiAaLxVwnw,4752
+dbt_common/events/functions.py,sha256=R3DuyNy2TqOwNyMACVOYtiG1NjZ9FQa6FWKkzMeMCBY,4767
 dbt_common/events/helpers.py,sha256=CfsWwNDjsLJkPIgOtAfuLEnZ3rGUKeYsH8aDtCW12OA,410
 dbt_common/events/interfaces.py,sha256=hEDeDoB0FW2RYHVZBG7gebEt_mUVBzkn1yPubpaxs-s,147
 dbt_common/events/logger.py,sha256=iBxMhFhAo8wL4NA4Z31pf644I0tsCOWIrt-k4d7EzaY,6760
@@ -57,7 +57,7 @@ dbt_common/utils/encoding.py,sha256=6_kSY2FvGNYMg7oX7PrbvVioieydih3Kl7Ii802LaHI,
 dbt_common/utils/executor.py,sha256=pNY0UbPlwQmTE69Vt_Rj91YGCIOEaqeYU3CjAds0T70,2454
 dbt_common/utils/formatting.py,sha256=JUn5rzJ-uajs9wPCN0-f2iRFY1pOJF5YjTD9dERuLoc,165
 dbt_common/utils/jinja.py,sha256=JXgNmJArGGy0h7qkbNLA3zaEQmoF1CxsNBYTlIwFXDw,1101
-dbt_common-1.15.0.dist-info/METADATA,sha256=gcyH_sfjldI5SpDsEGDig-tUAVMVANrqNEmN0jun2c0,4895
-dbt_common-1.15.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dbt_common-1.15.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dbt_common-1.15.0.dist-info/RECORD,,
+dbt_common-1.17.0.dist-info/METADATA,sha256=CxILo-M8pIxC0v3avF2ekgOrxwBrC_BHv5bD8kXKyk8,4895
+dbt_common-1.17.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dbt_common-1.17.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dbt_common-1.17.0.dist-info/RECORD,,