dbt-common 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

dbt_common/__about__.py

@@ -1 +1 @@
-version = "1.2.0"
+version = "1.4.0"

dbt_common/contracts/metadata.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
-from typing import Dict, Optional, Union
+from typing import Dict, Optional, Union, NamedTuple
 
 from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.utils.formatting import lowercase
 
 
 @dataclass
@@ -24,3 +25,35 @@ class TableMetadata(dbtClassMixin):
     database: Optional[str] = None
     comment: Optional[str] = None
     owner: Optional[str] = None
+
+
+CatalogKey = NamedTuple(
+    "CatalogKey", [("database", Optional[str]), ("schema", str), ("name", str)]
+)
+
+
+@dataclass
+class ColumnMetadata(dbtClassMixin):
+    type: str
+    index: int
+    name: str
+    comment: Optional[str] = None
+
+
+ColumnMap = Dict[str, ColumnMetadata]
+
+
+@dataclass
+class CatalogTable(dbtClassMixin):
+    metadata: TableMetadata
+    columns: ColumnMap
+    stats: StatsDict
+    # the same table with two unique IDs will just be listed two times
+    unique_id: Optional[str] = None
+
+    def key(self) -> CatalogKey:
+        return CatalogKey(
+            lowercase(self.metadata.database),
+            self.metadata.schema.lower(),
+            self.metadata.name.lower(),
+        )

dbt_common/record.py

@@ -9,6 +9,8 @@ import functools
 import dataclasses
 import json
 import os
+
+from deepdiff import DeepDiff  # type: ignore
 from enum import Enum
 from typing import Any, Dict, List, Mapping, Optional, Type
 
@@ -51,15 +53,68 @@ class Record:
 
 
 class Diff:
-    """Marker class for diffs?"""
+    def __init__(self, current_recording_path: str, previous_recording_path: str) -> None:
+        self.current_recording_path = current_recording_path
+        self.previous_recording_path = previous_recording_path
+
+    def diff_query_records(self, current: List, previous: List) -> Dict[str, Any]:
+        # some of the table results are returned as a stringified list of dicts that don't
+        # diff because order isn't consistent. convert it into a list of dicts so it can
+        # be diffed ignoring order
+
+        for i in range(len(current)):
+            if current[i].get("result").get("table") is not None:
+                current[i]["result"]["table"] = json.loads(current[i]["result"]["table"])
+        for i in range(len(previous)):
+            if previous[i].get("result").get("table") is not None:
+                previous[i]["result"]["table"] = json.loads(previous[i]["result"]["table"])
+
+        return DeepDiff(previous, current, ignore_order=True, verbose_level=2)
+
+    def diff_env_records(self, current: List, previous: List) -> Dict[str, Any]:
+        # The mode and filepath may change.  Ignore them.
+
+        exclude_paths = [
+            "root[0]['result']['env']['DBT_RECORDER_FILE_PATH']",
+            "root[0]['result']['env']['DBT_RECORDER_MODE']",
+        ]
+
+        return DeepDiff(
+            previous, current, ignore_order=True, verbose_level=2, exclude_paths=exclude_paths
+        )
+
+    def diff_default(self, current: List, previous: List) -> Dict[str, Any]:
+        return DeepDiff(previous, current, ignore_order=True, verbose_level=2)
+
+    def calculate_diff(self) -> Dict[str, Any]:
+        with open(self.current_recording_path) as current_recording:
+            current_dct = json.load(current_recording)
+
+        with open(self.previous_recording_path) as previous_recording:
+            previous_dct = json.load(previous_recording)
+
+        diff = {}
+        for record_type in current_dct:
+            if record_type == "QueryRecord":
+                diff[record_type] = self.diff_query_records(
+                    current_dct[record_type], previous_dct[record_type]
+                )
+            elif record_type == "GetEnvRecord":
+                diff[record_type] = self.diff_env_records(
+                    current_dct[record_type], previous_dct[record_type]
+                )
+            else:
+                diff[record_type] = self.diff_default(
+                    current_dct[record_type], previous_dct[record_type]
+                )
 
-    pass
+        return diff
 
 
 class RecorderMode(Enum):
     RECORD = 1
     REPLAY = 2
-    RECORD_QUERIES = 3
+    DIFF = 3  # records and does diffing
 
 
 class Recorder:
@@ -67,15 +122,31 @@ class Recorder:
     _record_name_by_params_name: Dict[str, str] = {}
 
     def __init__(
-        self, mode: RecorderMode, types: Optional[List], recording_path: Optional[str] = None
+        self,
+        mode: RecorderMode,
+        types: Optional[List],
+        current_recording_path: str = "recording.json",
+        previous_recording_path: Optional[str] = None,
     ) -> None:
         self.mode = mode
         self.types = types
         self._records_by_type: Dict[str, List[Record]] = {}
         self._replay_diffs: List["Diff"] = []
+        self.diff: Diff
+        self.previous_recording_path = previous_recording_path
+        self.current_recording_path = current_recording_path
+
+        if self.previous_recording_path is not None and self.mode in (
+            RecorderMode.REPLAY,
+            RecorderMode.DIFF,
+        ):
+            self.diff = Diff(
+                current_recording_path=self.current_recording_path,
+                previous_recording_path=self.previous_recording_path,
+            )
 
-        if recording_path is not None:
-            self._records_by_type = self.load(recording_path)
+            if self.mode == RecorderMode.REPLAY:
+                self._records_by_type = self.load(self.previous_recording_path)
 
     @classmethod
     def register_record_type(cls, rec_type) -> Any:
@@ -101,8 +172,8 @@ class Recorder:
 
         return match
 
-    def write(self, file_name) -> None:
-        with open(file_name, "w") as file:
+    def write(self) -> None:
+        with open(self.current_recording_path, "w") as file:
             json.dump(self._to_dict(), file)
 
     def _to_dict(self) -> Dict:
@@ -143,19 +214,19 @@ class Recorder:
 
     def write_diffs(self, diff_file_name) -> None:
         json.dump(
-            self._replay_diffs,
+            self.diff.calculate_diff(),
             open(diff_file_name, "w"),
         )
 
     def print_diffs(self) -> None:
-        print(repr(self._replay_diffs))
+        print(repr(self.diff.calculate_diff()))
 
 
 def get_record_mode_from_env() -> Optional[RecorderMode]:
     """
     Get the record mode from the environment variables.
 
-    If the mode is not set to 'RECORD' or 'REPLAY', return None.
+    If the mode is not set to 'RECORD', 'DIFF' or 'REPLAY', return None.
     Expected format: 'DBT_RECORDER_MODE=RECORD'
     """
     record_mode = os.environ.get("DBT_RECORDER_MODE")
@@ -165,6 +236,9 @@ def get_record_mode_from_env() -> Optional[RecorderMode]:
 
     if record_mode.lower() == "record":
         return RecorderMode.RECORD
+    # diffing requires a file path, otherwise treat as noop
+    elif record_mode.lower() == "diff" and os.environ.get("DBT_RECORDER_FILE_PATH") is not None:
+        return RecorderMode.DIFF
     # replaying requires a file path, otherwise treat as noop
     elif record_mode.lower() == "replay" and os.environ.get("DBT_RECORDER_FILE_PATH") is not None:
         return RecorderMode.REPLAY
@@ -190,6 +264,15 @@ def get_record_types_from_env() -> Optional[List]:
     return record_types_str.split(",")
 
 
+def get_record_types_from_dict(fp: str) -> List:
+    """
+    Get the record subset from the dict.
+    """
+    with open(fp) as file:
+        loaded_dct = json.load(file)
+    return list(loaded_dct.keys())
+
+
 def record_function(record_type, method=False, tuple_result=False):
     def record_function_inner(func_to_record):
         # To avoid runtime overhead and other unpleasantness, we only apply the

{dbt_common-1.2.0.dist-info → dbt_common-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-common
-Version: 1.2.0
+Version: 1.4.0
 Summary: The shared common utilities that dbt-core and adapter implementations use
 Project-URL: Homepage, https://github.com/dbt-labs/dbt-common
 Project-URL: Repository, https://github.com/dbt-labs/dbt-common.git
@@ -26,6 +26,7 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.8
 Requires-Dist: agate<1.10,>=1.7.0
 Requires-Dist: colorama<0.5,>=0.3.9
+Requires-Dist: deepdiff<8.0,>=7.0
 Requires-Dist: isodate<0.7,>=0.6
 Requires-Dist: jinja2<4,>=3.1.3
 Requires-Dist: jsonschema<5.0,>=4.0

{dbt_common-1.2.0.dist-info → dbt_common-1.4.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dbt_common/__about__.py,sha256=-XvoSdwL-0go8ZX6VN74e-D6dwG0z-0p7vL5MO0Sz5s,18
+dbt_common/__about__.py,sha256=0kccNYBMuNA3PIhlESWmh8xP1TWpNtIEzS0d-x80SC0,18
 dbt_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbt_common/constants.py,sha256=-Y5DIL1SDPQWtlCNizXRYxFgbx1D7LaLs1ysamvGMRk,278
 dbt_common/context.py,sha256=BhgT7IgyvpZHEtIdFVVuBBBX5LuU7obXT7NvIPeuD2g,1760
@@ -6,7 +6,7 @@ dbt_common/dataclass_schema.py,sha256=t3HGD0oXTSjitctuCVHv3iyq5BT3jxoSxv_VGkrJlE
 dbt_common/helper_types.py,sha256=NoxqGFAq9bOjh7rqtz_eepXAxk20n3mmW_gUVpnMyYU,3901
 dbt_common/invocation.py,sha256=Zw8jRPn75oi2VrUD6qGvaCDtSyIfqm5pJlPpRjs3s1E,202
 dbt_common/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dbt_common/record.py,sha256=fwp8Q2x0UiB9fHbMFkmOlAgKLst1dhDxQpSSsd04aDw,8204
+dbt_common/record.py,sha256=_P-OZWKQwGmq6nTC4r2oQDyh89gi-VGC2gdok6TywhI,11765
 dbt_common/semver.py,sha256=2zoZYCQ7PfswqslT2NHuMGgPGMuMuX-yRThVoqfDWQU,13954
 dbt_common/tests.py,sha256=6lC_JuRtoYO6cbAF8-R5aTM4HtQiM_EH8X5m_97duGY,315
 dbt_common/ui.py,sha256=rc2TEM29raBFc_LXcg901pMDD07C2ohwp9qzkE-7pBY,2567
@@ -17,7 +17,7 @@ dbt_common/clients/jinja.py,sha256=i6VQ94FU4F6ZCQLHTxNSeGHmvyYSIe34nDhNkH6wO08,1
 dbt_common/clients/system.py,sha256=OOhRDWR5t0Ns3OhkqjPTNTtyl_RMRWPDHWCzDoFtgkA,23014
 dbt_common/contracts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbt_common/contracts/constraints.py,sha256=hyqTW2oPB1dfXWW388LWnL-EFdqTpQciKISH3CeLkro,1267
-dbt_common/contracts/metadata.py,sha256=d5zKvIR3s2j6_ZvafvlyNm5z6MVs8augT2jclbOSG-k,528
+dbt_common/contracts/metadata.py,sha256=K_M06Rue0wmrQhFP_mq3uvQszq10CIt93oGiAVgbRfE,1293
 dbt_common/contracts/util.py,sha256=RZpeEExSKdyFwTq7MM3rd1ZkAf11C7I-bgppUJ6SXOg,741
 dbt_common/contracts/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbt_common/contracts/config/base.py,sha256=jWLf6SBUy7wngYs0Z5Zmx1O1v86XRueYaABlZ0W2Bxc,8356
@@ -56,7 +56,7 @@ dbt_common/utils/encoding.py,sha256=6_kSY2FvGNYMg7oX7PrbvVioieydih3Kl7Ii802LaHI,
 dbt_common/utils/executor.py,sha256=Zyzd1wML3aN-iYn9ZG2Gc_jj5vknmvQNyH-c0RaPIpo,2446
 dbt_common/utils/formatting.py,sha256=JUn5rzJ-uajs9wPCN0-f2iRFY1pOJF5YjTD9dERuLoc,165
 dbt_common/utils/jinja.py,sha256=XNfZHuZhLM_R_yPmzYojPm6bF7QOoxIjSWrkJRw6wks,965
-dbt_common-1.2.0.dist-info/METADATA,sha256=7rA7to-AQn0UQvVJVAy14wx7uhcyB8D_1wHmTTtEHDo,5264
-dbt_common-1.2.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-dbt_common-1.2.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-dbt_common-1.2.0.dist-info/RECORD,,
+dbt_common-1.4.0.dist-info/METADATA,sha256=QH1OkqWWMGMTDUDSrqHmEnEvHs0akdlpDU-5asSCUD8,5298
+dbt_common-1.4.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+dbt_common-1.4.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dbt_common-1.4.0.dist-info/RECORD,,