dbt-common 1.1.0__tar.gz → 1.3.0__tar.gz

{dbt_common-1.1.0 → dbt_common-1.3.0}/.gitignore

@@ -157,4 +157,4 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/

{dbt_common-1.1.0 → dbt_common-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-common
-Version: 1.1.0
+Version: 1.3.0
 Summary: The shared common utilities that dbt-core and adapter implementations use
 Project-URL: Homepage, https://github.com/dbt-labs/dbt-common
 Project-URL: Repository, https://github.com/dbt-labs/dbt-common.git

dbt_common-1.3.0/dbt_common/__about__.py

@@ -0,0 +1 @@
+version = "1.3.0"

dbt_common-1.3.0/dbt_common/contracts/metadata.py

@@ -0,0 +1,59 @@
+from dataclasses import dataclass
+from typing import Dict, Optional, Union, NamedTuple
+
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.utils.formatting import lowercase
+
+
+@dataclass
+class StatsItem(dbtClassMixin):
+    id: str
+    label: str
+    value: Union[bool, str, float, None]
+    include: bool
+    description: Optional[str] = None
+
+
+StatsDict = Dict[str, StatsItem]
+
+
+@dataclass
+class TableMetadata(dbtClassMixin):
+    type: str
+    schema: str
+    name: str
+    database: Optional[str] = None
+    comment: Optional[str] = None
+    owner: Optional[str] = None
+
+
+CatalogKey = NamedTuple(
+    "CatalogKey", [("database", Optional[str]), ("schema", str), ("name", str)]
+)
+
+
+@dataclass
+class ColumnMetadata(dbtClassMixin):
+    type: str
+    index: int
+    name: str
+    comment: Optional[str] = None
+
+
+ColumnMap = Dict[str, ColumnMetadata]
+
+
+@dataclass
+class CatalogTable(dbtClassMixin):
+    metadata: TableMetadata
+    columns: ColumnMap
+    stats: StatsDict
+    # the same table with two unique IDs will just be listed two times
+    unique_id: Optional[str] = None
+
+    def key(self) -> CatalogKey:
+        return CatalogKey(
+            lowercase(self.metadata.database),
+            self.metadata.schema.lower(),
+            self.metadata.name.lower(),
+        )

{dbt_common-1.1.0 → dbt_common-1.3.0}/dbt_common/record.py

@@ -59,14 +59,18 @@ class Diff:
 class RecorderMode(Enum):
     RECORD = 1
     REPLAY = 2
+    RECORD_QUERIES = 3
 
 
 class Recorder:
     _record_cls_by_name: Dict[str, Type] = {}
     _record_name_by_params_name: Dict[str, str] = {}
 
-    def __init__(self, mode: RecorderMode, recording_path: Optional[str] = None) -> None:
+    def __init__(
+        self, mode: RecorderMode, types: Optional[List], recording_path: Optional[str] = None
+    ) -> None:
         self.mode = mode
+        self.types = types
         self._records_by_type: Dict[str, List[Record]] = {}
         self._replay_diffs: List["Diff"] = []
 
@@ -118,13 +122,14 @@ class Recorder:
         records_by_type: Dict[str, List[Record]] = {}
 
         for record_type_name in loaded_dct:
+            # TODO: this breaks with QueryRecord on replay since it's
+            # not in common so isn't part of cls._record_cls_by_name yet
             record_cls = cls._record_cls_by_name[record_type_name]
             rec_list = []
             for record_dct in loaded_dct[record_type_name]:
                 rec = record_cls.from_dict(record_dct)
                 rec_list.append(rec)  # type: ignore
             records_by_type[record_type_name] = rec_list
-
         return records_by_type
 
     def expect_record(self, params: Any) -> Any:
@@ -147,17 +152,44 @@ class Recorder:
 
 
 def get_record_mode_from_env() -> Optional[RecorderMode]:
-    replay_val = os.environ.get("DBT_REPLAY")
-    if replay_val is not None and replay_val != "0" and replay_val.lower() != "false":
-        return RecorderMode.REPLAY
+    """
+    Get the record mode from the environment variables.
+
+    If the mode is not set to 'RECORD' or 'REPLAY', return None.
+    Expected format: 'DBT_RECORDER_MODE=RECORD'
+    """
+    record_mode = os.environ.get("DBT_RECORDER_MODE")
 
-    record_val = os.environ.get("DBT_RECORD")
-    if record_val is not None and record_val != "0" and record_val.lower() != "false":
+    if record_mode is None:
+        return None
+
+    if record_mode.lower() == "record":
         return RecorderMode.RECORD
+    # replaying requires a file path, otherwise treat as noop
+    elif record_mode.lower() == "replay" and os.environ.get("DBT_RECORDER_FILE_PATH") is not None:
+        return RecorderMode.REPLAY
 
+    # if you don't specify record/replay it's a noop
     return None
 
 
+def get_record_types_from_env() -> Optional[List]:
+    """
+    Get the record subset from the environment variables.
+
+    If no types are provided, there will be no filtering.
+    Invalid types will be ignored.
+    Expected format: 'DBT_RECORDER_TYPES=QueryRecord,FileLoadRecord,OtherRecord'
+    """
+    record_types_str = os.environ.get("DBT_RECORDER_TYPES")
+
+    # if all is specified we don't want any type filtering
+    if record_types_str is None or record_types_str.lower == "all":
+        return None
+
+    return record_types_str.split(",")
+
+
 def record_function(record_type, method=False, tuple_result=False):
     def record_function_inner(func_to_record):
         # To avoid runtime overhead and other unpleasantness, we only apply the
@@ -176,6 +208,9 @@ def record_function(record_type, method=False, tuple_result=False):
             if recorder is None:
                 return func_to_record(*args, **kwargs)
 
+            if recorder.types is not None and record_type.__name__ not in recorder.types:
+                return func_to_record(*args, **kwargs)
+
             # For methods, peel off the 'self' argument before calling the
             # params constructor.
             param_args = args[1:] if method else args

{dbt_common-1.1.0 → dbt_common-1.3.0}/docs/guides/record_replay.md

@@ -28,7 +28,22 @@ Note also the `LoadFileRecord` class passed as a parameter to this decorator. Th
   
 The final detail needed is to define the classes specified by `params_cls` and `result_cls`, which must be dataclasses with properties whose order and names correspond to the parameters passed to the recorded function. In this case those are the `LoadFileParams` and `LoadFileResult` classes, respectively.
 
-With these decorators applied and classes defined, dbt is able to record all file access during a run, and mock out the accesses during replay, isolating dbt from actually loading files. At least it would if dbt only used this function for all file access, which is only mostly true. We hope to continue improving the usefulness of this mechanism by adding more recorded functions and routing more operations through them.  
+With these decorators applied and classes defined, dbt is able to record all file access during a run, and mock out the accesses during replay, isolating dbt from actually loading files. At least it would if dbt only used this function for all file access, which is only mostly true. We hope to continue improving the usefulness of this mechanism by adding more recorded functions and routing more operations through them.
+
+## How to record/replay
+If `DBT_RECORDER_MODE` is not `replay` or `record`, case insensitive, this is a no-op.  Invalid values are ignored and do not throw exceptions.
+
+`DBT_RECODER_TYPES` is optional.  It indicates which types to filter the results by and expects a list of strings values for the `Record` subclasses.  Any invalid types will be ignored.  `all` is a valid type and behaves the same as not populating the env var.
+
+
+```bash
+DBT_RECORDER_MODE=record DBT_RECODER_TYPES=QueryRecord,GetEnvRecord dbt run
+```
+
+replay need the file to replay
+```bash
+DBT_RECORDER_MODE=replay DBT_RECORDER_FILE_PATH=recording.json dbt run
+```
 
 ## Final Thoughts
   

dbt_common-1.1.0/dbt_common/__about__.py

@@ -1 +0,0 @@
-version = "1.1.0"