dvt-core 1.11.0b4__py3-none-any.whl

dvt/event_time/event_time.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+
+from dateutil.relativedelta import relativedelta
+from dvt.artifacts.resources.types import BatchSize
+
+from dbt_common.exceptions import DbtRuntimeError
+
+
+def offset_timestamp(timestamp=datetime, batch_size=BatchSize, offset=int) -> datetime:
+    """Offsets the passed in timestamp based on the batch_size and offset.
+
+    Note: THIS IS DIFFERENT FROM MicrobatchBuilder.offset_timestamp. That function first
+    `truncates` the timestamp, and then does delta addition subtraction from there. This
+    function _doesn't_ truncate the timestamp and uses `relativedelta` for specific edge
+    case handling (months, years), which may produce different results than the delta math
+    done in `MicrobatchBuilder.offset_timestamp`
+
+    Examples
+    2024-09-17 16:06:00 + Batchsize.hour -1 -> 2024-09-17 15:06:00
+    2024-09-17 16:06:00 + Batchsize.hour +1 -> 2024-09-17 17:06:00
+    2024-09-17 16:06:00 + Batchsize.day -1 -> 2024-09-16 16:06:00
+    2024-09-17 16:06:00 + Batchsize.day +1 -> 2024-09-18 16:06:00
+    2024-09-17 16:06:00 + Batchsize.month -1 -> 2024-08-17 16:06:00
+    2024-09-17 16:06:00 + Batchsize.month +1 -> 2024-10-17 16:06:00
+    2024-09-17 16:06:00 + Batchsize.year -1 -> 2023-09-17 16:06:00
+    2024-09-17 16:06:00 + Batchsize.year +1 -> 2025-09-17 16:06:00
+    2024-01-31 16:06:00 + Batchsize.month +1 -> 2024-02-29 16:06:00
+    2024-02-29 16:06:00 + Batchsize.year +1 -> 2025-02-28 16:06:00
+    """
+
+    if batch_size == BatchSize.hour:
+        return timestamp + relativedelta(hours=offset)
+    elif batch_size == BatchSize.day:
+        return timestamp + relativedelta(days=offset)
+    elif batch_size == BatchSize.month:
+        return timestamp + relativedelta(months=offset)
+    elif batch_size == BatchSize.year:
+        return timestamp + relativedelta(years=offset)
+    else:
+        raise DbtRuntimeError(f"Unhandled batch_size '{batch_size}'")

dvt/event_time/sample_window.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+import pytz
+from attr import dataclass
+from dvt.artifacts.resources.types import BatchSize
+from dvt.event_time.event_time import offset_timestamp
+
+from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.exceptions import DbtRuntimeError
+
+
+@dataclass
+class SampleWindow(dbtClassMixin):
+    start: datetime
+    end: datetime
+
+    def __post_serialize__(self, data, context):
+        # This is insane, but necessary, I apologize. Mashumaro handles the
+        # dictification of this class via a compile time generated `to_dict`
+        # method based off of the _typing_ of th class. By default `datetime`
+        # types are converted to strings. We don't want that, we want them to
+        # stay datetimes.
+        # Note: This is safe because the `SampleWindow` isn't part of the artifact
+        # and thus doesn't get written out.
+        new_data = super().__post_serialize__(data, context)
+        new_data["start"] = self.start
+        new_data["end"] = self.end
+        return new_data
+
+    @classmethod
+    def from_relative_string(cls, relative_string: str) -> SampleWindow:
+        end = datetime.now(tz=pytz.UTC)
+
+        relative_window = relative_string.split(" ")
+        if len(relative_window) != 2:
+            raise DbtRuntimeError(
+                f"Cannot load SAMPLE_WINDOW from '{relative_string}'. Must be of form 'DAYS_INT GRAIN_SIZE'."
+            )
+
+        try:
+            lookback = int(relative_window[0])
+        except Exception:
+            raise DbtRuntimeError(f"Unable to convert '{relative_window[0]}' to an integer.")
+
+        try:
+            batch_size_string = relative_window[1].lower().rstrip("s")
+            batch_size = BatchSize[batch_size_string]
+        except Exception:
+            grains = [size.value for size in BatchSize]
+            grain_plurals = [BatchSize.plural(size) for size in BatchSize]
+            valid_grains = grains + grain_plurals
+            raise DbtRuntimeError(
+                f"Invalid grain size '{relative_window[1]}'. Must be one of {valid_grains}."
+            )
+
+        start = offset_timestamp(timestamp=end, batch_size=batch_size, offset=-1 * lookback)
+
+        return cls(start=start, end=end)

dvt/events/__init__.py

@@ -0,0 +1,16 @@
+from typing import Any, Dict, Set
+
+import dvt.events.types as core_dbt_event_types
+
+import dbt.adapters.events.types as adapter_dbt_event_types
+import dbt_common.events.types as dbt_event_types
+
+ALL_EVENT_TYPES: Dict[str, Any] = {
+    **dbt_event_types.__dict__,
+    **core_dbt_event_types.__dict__,
+    **adapter_dbt_event_types.__dict__,
+}
+
+ALL_EVENT_NAMES: Set[str] = set(
+    [name for name, cls in ALL_EVENT_TYPES.items() if isinstance(cls, type)]
+)

dvt/events/base_types.py

@@ -0,0 +1,37 @@
+from dvt.events import core_types_pb2
+
+from dbt_common.events.base_types import BaseEvent
+from dbt_common.events.base_types import DebugLevel as CommonDebugLevel
+from dbt_common.events.base_types import DynamicLevel as CommonDyanicLevel
+from dbt_common.events.base_types import ErrorLevel as CommonErrorLevel
+from dbt_common.events.base_types import InfoLevel as CommonInfoLevel
+from dbt_common.events.base_types import TestLevel as CommonTestLevel
+from dbt_common.events.base_types import WarnLevel as CommonWarnLevel
+
+
+class CoreBaseEvent(BaseEvent):
+    PROTO_TYPES_MODULE = core_types_pb2
+
+
+class DynamicLevel(CommonDyanicLevel, CoreBaseEvent):
+    pass
+
+
+class TestLevel(CommonTestLevel, CoreBaseEvent):
+    pass
+
+
+class DebugLevel(CommonDebugLevel, CoreBaseEvent):
+    pass
+
+
+class InfoLevel(CommonInfoLevel, CoreBaseEvent):
+    pass
+
+
+class WarnLevel(CommonWarnLevel, CoreBaseEvent):
+    pass
+
+
+class ErrorLevel(CommonErrorLevel, CoreBaseEvent):
+    pass

dvt/events/core_types_pb2.py

@@ -0,0 +1,2 @@
+# preserving import path during dbtlabs.proto refactor
+from dbtlabs.proto.public.v1.fields.core_types_pb2 import *  # noqa

dvt/events/logging.py

@@ -0,0 +1,109 @@
+import os
+from functools import partial
+from typing import Callable, List
+
+from dvt.tracking import track_behavior_change_warn
+
+from dbt_common.events.base_types import EventLevel, EventMsg
+from dbt_common.events.event_manager_client import (
+    add_callback_to_manager,
+    add_logger_to_manager,
+    cleanup_event_logger,
+    get_event_manager,
+)
+from dbt_common.events.functions import (
+    env_scrubber,
+    get_capture_stream,
+    get_stdout_config,
+    make_log_dir_if_missing,
+)
+from dbt_common.events.logger import LineFormat, LoggerConfig
+from dbt_common.invocation import get_invocation_id
+
+# These are the logging events issued by the "clean" command,
+# where we can't count on having a log directory. We've removed
+# the "class" flags on the events in types.py. If necessary we
+# could still use class or method flags, but we'd have to get
+# the type class from the msg and then get the information from the class.
+_NOFILE_CODES = ["Z012", "Z013", "Z014", "Z015"]
+
+
+def _line_format_from_str(format_str: str, default: LineFormat) -> LineFormat:
+    if format_str == "text":
+        return LineFormat.PlainText
+    elif format_str == "debug":
+        return LineFormat.DebugText
+    elif format_str == "json":
+        return LineFormat.Json
+
+    return default
+
+
+def _get_logfile_config(
+    log_path: str,
+    use_colors: bool,
+    line_format: LineFormat,
+    level: EventLevel,
+    log_file_max_bytes: int,
+    log_cache_events: bool = False,
+) -> LoggerConfig:
+    return LoggerConfig(
+        name="file_log",
+        line_format=line_format,
+        use_colors=use_colors,
+        level=level,  # File log is *always* debug level
+        scrubber=env_scrubber,
+        filter=partial(_logfile_filter, log_cache_events, line_format),
+        invocation_id=get_invocation_id(),
+        output_file_name=log_path,
+        output_file_max_bytes=log_file_max_bytes,
+    )
+
+
+def _logfile_filter(log_cache_events: bool, line_format: LineFormat, msg: EventMsg) -> bool:
+    return msg.info.code not in _NOFILE_CODES and not (
+        msg.info.name in ["CacheAction", "CacheDumpGraph"] and not log_cache_events
+    )
+
+
+def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = []) -> None:
+    cleanup_event_logger()
+    make_log_dir_if_missing(flags.LOG_PATH)
+    event_manager = get_event_manager()
+    event_manager.callbacks = callbacks.copy()
+    add_callback_to_manager(track_behavior_change_warn)
+
+    if flags.LOG_LEVEL != "none":
+        line_format = _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText)
+        log_level = (
+            EventLevel.ERROR
+            if flags.QUIET
+            else EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL)
+        )
+        console_config = get_stdout_config(
+            line_format,
+            flags.USE_COLORS,
+            log_level,
+            flags.LOG_CACHE_EVENTS,
+        )
+
+        if get_capture_stream():
+            # Create second stdout logger to support test which want to know what's
+            # being sent to stdout.
+            console_config.output_stream = get_capture_stream()
+        add_logger_to_manager(console_config)
+
+    if flags.LOG_LEVEL_FILE != "none":
+        # create and add the file logger to the event manager
+        log_file = os.path.join(flags.LOG_PATH, "dbt.log")
+        log_file_format = _line_format_from_str(flags.LOG_FORMAT_FILE, LineFormat.DebugText)
+        log_level_file = EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL_FILE)
+        add_logger_to_manager(
+            _get_logfile_config(
+                log_file,
+                flags.USE_COLORS_FILE,
+                log_file_format,
+                log_level_file,
+                flags.LOG_FILE_MAX_BYTES,
+            )
+        )