dsgrid-toolkit 0.3.3__cp313-cp313-win_amd64.whl

dsgrid/dimension/time.py

@@ -0,0 +1,352 @@
+"""Dimensions related to time"""
+
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+import logging
+from pydantic import Field
+from enum import Enum
+
+
+from dsgrid.data_models import DSGEnum, EnumValue, DSGBaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class TimeDimensionType(DSGEnum):
+    """Defines the supported time formats in the load data."""
+
+    DATETIME = "datetime"
+    ANNUAL = "annual"
+    REPRESENTATIVE_PERIOD = "representative_period"
+    DATETIME_EXTERNAL_TZ = "datetime_external_tz"
+    INDEX = "index"
+    NOOP = "noop"
+
+
+class TimeZoneFormat(str, Enum):
+    """Defines the time format of the datetime config model"""
+
+    ALIGNED_IN_ABSOLUTE_TIME = "aligned_in_absolute_time"
+    ALIGNED_IN_CLOCK_TIME = "aligned_in_clock_time"
+    LOCAL_AS_STRINGS = "local_as_strings"
+
+
+class RepresentativePeriodFormat(DSGEnum):
+    """Defines the supported formats for representative period data."""
+
+    # All instances of this Enum must declare frequency.
+    # This Enum may be replaced by a generic implementation in order to support a large
+    # number of permutations (seasons, weekend day vs week day, sub-hour time, etc).
+
+    ONE_WEEK_PER_MONTH_BY_HOUR = EnumValue(
+        value="one_week_per_month_by_hour",
+        frequency=timedelta(hours=1),
+        description="load_data columns use 'month', 'day_of_week', 'hour' to specify time",
+    )
+    ONE_WEEKDAY_DAY_AND_ONE_WEEKEND_DAY_PER_MONTH_BY_HOUR = EnumValue(
+        value="one_weekday_day_and_one_weekend_day_per_month_by_hour",
+        frequency=timedelta(hours=1),
+        description="load_data columns use 'month', 'hour', 'is_weekday' to specify time",
+    )
+
+
+class LeapDayAdjustmentType(DSGEnum):
+    """Leap day adjustment enum types"""
+
+    DROP_DEC31 = EnumValue(
+        value="drop_dec31",
+        description="To adjust for leap years, December 31st timestamps and data get dropped.",
+    )
+    DROP_FEB29 = EnumValue(
+        value="drop_feb29",
+        description="Feburary 29th timestamps and data are dropped. Currently not yet supported by dsgrid.",
+    )
+    DROP_JAN1 = EnumValue(
+        value="drop_jan1",
+        description="To adjust for leap years, January 1st timestamps and data get dropped.",
+    )
+    NONE = EnumValue(value="none", description="No leap day adjustment made.")
+
+
+class DaylightSavingSpringForwardType(DSGEnum):
+    """Daylight saving spring forward adjustment enum types"""
+
+    DROP = EnumValue(
+        value="drop",
+        description="Drop timestamp(s) and associated data for the spring forward hour (2AM in March)",
+    )
+    NONE = EnumValue(value="none", description="No daylight saving adjustment for data.")
+
+
+class DaylightSavingFallBackType(DSGEnum):
+    """Daylight saving fall back adjustment enum types"""
+
+    INTERPOLATE = EnumValue(
+        value="interpolate",
+        description="Fill data by interpolating between the left and right edges of the dataframe.",
+    )
+    DUPLICATE = EnumValue(
+        value="duplicate",
+        description="Fill data by duplicating the fall-back hour (1AM in November)",
+    )
+    NONE = EnumValue(value="none", description="No daylight saving adjustment for data.")
+
+
+class TimeIntervalType(DSGEnum):
+    """Time interval enum types"""
+
+    # TODO: R2PD uses a different set; do we want to align?
+    # https://github.com/Smart-DS/R2PD/blob/master/R2PD/tshelpers.py#L15
+
+    PERIOD_ENDING = EnumValue(
+        value="period_ending",
+        description="A time interval that is period ending is coded by the end time. E.g., 2pm (with"
+        " freq=1h) represents a period of time between 1-2pm.",
+    )
+    PERIOD_BEGINNING = EnumValue(
+        value="period_beginning",
+        description="A time interval that is period beginning is coded by the beginning time. E.g.,"
+        " 2pm (with freq=01:00:00) represents a period of time between 2-3pm. This is the dsgrid"
+        " default.",
+    )
+    INSTANTANEOUS = EnumValue(
+        value="instantaneous",
+        description="The time record value represents measured, instantaneous time",
+    )
+
+
+class MeasurementType(DSGEnum):
+    """Time value measurement enum types"""
+
+    MEAN = EnumValue(
+        value="mean",
+        description="Data values represent the average value in a time range",
+    )
+    MIN = EnumValue(
+        value="min",
+        description="Data values represent the minimum value in a time range",
+    )
+    MAX = EnumValue(
+        value="max",
+        description="Data values represent the maximum value in a time range",
+    )
+    MEASURED = EnumValue(
+        value="measured",
+        description="Data values represent the measured value at that reported time",
+    )
+    TOTAL = EnumValue(
+        value="total",
+        description="Data values represent the sum of values in a time range",
+    )
+
+
+class DaylightSavingAdjustmentModel(DSGBaseModel):
+    """Defines how to drop and add data along with timestamps to convert standard time
+    load profiles to clock time"""
+
+    spring_forward_hour: DaylightSavingSpringForwardType = Field(
+        title="spring_forward_hour",
+        description="Data adjustment for spring forward hour (a 2AM in March)",
+        default=DaylightSavingSpringForwardType.NONE,
+        json_schema_extra={
+            "options": DaylightSavingSpringForwardType.format_descriptions_for_docs(),
+        },
+    )
+
+    fall_back_hour: DaylightSavingFallBackType = Field(
+        title="fall_back_hour",
+        description="Data adjustment for spring forward hour (a 2AM in November)",
+        default=DaylightSavingFallBackType.NONE,
+        json_schema_extra={
+            "options": DaylightSavingFallBackType.format_descriptions_for_docs(),
+        },
+    )
+
+
+class TimeBasedDataAdjustmentModel(DSGBaseModel):
+    """Defines how data needs to be adjusted with respect to time.
+    For leap day adjustment, up to one full day of timestamps and data are dropped.
+    For daylight savings, the dataframe is adjusted alongside the timestamps.
+    This is useful when the load profiles are modeled in standard time and
+    need to be converted to get clock time load profiles.
+    """
+
+    leap_day_adjustment: LeapDayAdjustmentType = Field(
+        default=LeapDayAdjustmentType.NONE,
+        title="leap_day_adjustment",
+        description="Leap day adjustment method applied to time data. The dsgrid default is None, "
+        "i.e., no adjustment made to leap years. Adjustments are made to leap years only.",
+    )
+    daylight_saving_adjustment: DaylightSavingAdjustmentModel = Field(
+        title="daylight_saving_adjustment",
+        description="Daylight saving adjustment method applied to time data",
+        default=DaylightSavingAdjustmentModel(
+            spring_forward_hour=DaylightSavingSpringForwardType.NONE,
+            fall_back_hour=DaylightSavingFallBackType.NONE,
+        ),
+    )
+
+
+class DatetimeRange:
+    def __init__(
+        self,
+        start,
+        end,
+        frequency,
+        time_based_data_adjustment: TimeBasedDataAdjustmentModel | None = None,
+    ):
+        if time_based_data_adjustment is None:
+            time_based_data_adjustment = TimeBasedDataAdjustmentModel()
+        self.start = start
+        self.end = end
+        self.tzinfo = start.tzinfo
+        self.frequency = frequency
+        self.leap_day_adjustment = time_based_data_adjustment.leap_day_adjustment
+        self.dls_springforward_adjustment = (
+            time_based_data_adjustment.daylight_saving_adjustment.spring_forward_hour
+        )
+        self.dls_fallback_adjustment = (
+            time_based_data_adjustment.daylight_saving_adjustment.fall_back_hour
+        )
+
+    def __repr__(self):
+        return (
+            self.__class__.__qualname__
+            + f"(start={self.start}, end={self.end}, frequency={self.frequency}, "
+            + f"leap_day_adjustment={self.leap_day_adjustment}, "
+            + f"dls_springforward_adjustment={self.dls_springforward_adjustment}, "
+            + f"dls_fallback_adjustment={self.dls_fallback_adjustment}."
+        )
+
+    def __str__(self):
+        return self.show_range()
+
+    def show_range(self, n_show=5):
+        output = self.list_time_range()
+        n_show = min(len(output) // 2, n_show)
+        n_head = ", ".join([str(x) for x in output[:n_show]])
+        n_tail = ", ".join([str(x) for x in output[-n_show:]])
+        return n_head + ",\n ... , \n" + n_tail
+
+    def _iter_timestamps(self):
+        """Return a generator of datetimes for a time range ('start' and 'end' times are inclusive).
+        There could be duplicates.
+
+        TODO: for future-selves, test functionality of LeapDayAdjustmentType in relation to TimeIntervalType to make sure drop behavior is expected.
+
+        Yields
+        ------
+        datetime
+
+        """
+        cur = self.start.to_pydatetime().astimezone(ZoneInfo("UTC"))
+        end = self.end.to_pydatetime().astimezone(ZoneInfo("UTC")) + self.frequency
+
+        while cur < end:
+            cur_tz = cur.astimezone(self.tzinfo)
+            cur_tz = adjust_timestamp_by_dst_offset(cur_tz, self.frequency)
+            month = cur_tz.month
+            day = cur_tz.day
+            if not (
+                self.leap_day_adjustment == LeapDayAdjustmentType.DROP_FEB29
+                and month == 2
+                and day == 29
+            ):
+                if not (
+                    self.leap_day_adjustment == LeapDayAdjustmentType.DROP_DEC31
+                    and month == 12
+                    and day == 31
+                ):
+                    if not (
+                        self.leap_day_adjustment == LeapDayAdjustmentType.DROP_JAN1
+                        and month == 1
+                        and day == 1
+                    ):
+                        yield cur_tz
+
+            cur += self.frequency
+
+    def list_time_range(self):
+        """Return a list of timestamps for a time range.
+
+        Returns
+        -------
+        list[datetime]
+        """
+        return list(self._iter_timestamps())
+
+
+class AnnualTimeRange(DatetimeRange):
+    def _iter_timestamps(self):
+        """
+        Return a list of years (datetime obj) on Jan 1st
+        Might be okay to not convert to UTC for iteration, since it's annual
+
+        """
+        start = self.start.to_pydatetime()
+        end = self.end.to_pydatetime()
+        tz = self.tzinfo
+        assert isinstance(self.frequency, int)
+        for year in range(start.year, end.year + self.frequency, self.frequency):
+            yield datetime(year=year, month=1, day=1, tzinfo=tz)
+
+
+class IndexTimeRange(DatetimeRange):
+    def __init__(
+        self,
+        start,
+        end,
+        frequency,
+        start_index,
+        time_based_data_adjustment: TimeBasedDataAdjustmentModel | None = None,
+    ):
+        super().__init__(
+            start, end, frequency, time_based_data_adjustment=time_based_data_adjustment
+        )
+        self.start_index = start_index
+
+    def _iter_timestamps(self):
+        cur = self.start.to_pydatetime().astimezone(ZoneInfo("UTC"))
+        cur_idx = self.start_index
+        end = (
+            self.end.to_pydatetime().astimezone(ZoneInfo("UTC")) + self.frequency
+        )  # to make end time inclusive
+
+        while cur < end:
+            cur_tz = cur.astimezone(self.tzinfo)
+            cur_tz = adjust_timestamp_by_dst_offset(cur_tz, self.frequency)
+            month = cur_tz.month
+            day = cur_tz.day
+            if not (
+                self.leap_day_adjustment == LeapDayAdjustmentType.DROP_FEB29
+                and month == 2
+                and day == 29
+            ):
+                if not (
+                    self.leap_day_adjustment == LeapDayAdjustmentType.DROP_DEC31
+                    and month == 12
+                    and day == 31
+                ):
+                    if not (
+                        self.leap_day_adjustment == LeapDayAdjustmentType.DROP_JAN1
+                        and month == 1
+                        and day == 1
+                    ):
+                        yield cur_idx
+            cur += self.frequency
+            cur_idx += 1
+
+
+def adjust_timestamp_by_dst_offset(timestamp, frequency):
+    """Reduce the timestamps within the daylight saving range by 1 hour.
+    Used to ensure that a time series at daily (or lower) frequency returns each day at the
+    same timestamp in prevailing time, an expected behavior in most standard libraries.
+    (e.g., ensure a time series can return 2018-03-11 00:00, 2018-03-12 00:00...
+    instead of 2018-03-11 00:00, 2018-03-12 01:00...)
+
+    """
+    if frequency < timedelta(hours=24):
+        return timestamp
+
+    offset = timestamp.dst() or timedelta(hours=0)
+    return timestamp - offset

dsgrid/dimension/time_utils.py

@@ -0,0 +1,103 @@
+"""Functions related to time"""
+
+from datetime import datetime
+
+import logging
+
+
+import pandas as pd
+
+from dsgrid.dimension.time import (
+    DatetimeRange,
+    TimeBasedDataAdjustmentModel,
+    TimeDimensionType,
+)
+from dsgrid.config.dimensions import TimeRangeModel, AnnualRangeModel
+
+
+logger = logging.getLogger(__name__)
+
+
+def build_annual_ranges(
+    time_ranges: list[AnnualRangeModel],
+    tz: str | None = None,
+) -> list[tuple[pd.Timestamp, pd.Timestamp, int]]:
+    ranges = []
+    for time_range in time_ranges:
+        start = datetime.strptime(time_range.start, time_range.str_format)
+        end = datetime.strptime(time_range.end, time_range.str_format)
+        assert isinstance(time_range.frequency, int)
+        freq = time_range.frequency
+        ranges.append((pd.Timestamp(start, tz=tz), pd.Timestamp(end, tz=tz), freq))
+
+    ranges.sort(key=lambda x: x[0])
+    return ranges
+
+
+def build_time_ranges(
+    time_ranges: list[TimeRangeModel],
+    tz: str | None = None,
+) -> list[tuple[pd.Timestamp, pd.Timestamp, pd.Timedelta]]:
+    ranges = []
+    for time_range in time_ranges:
+        start = datetime.strptime(time_range.start, time_range.str_format)
+        end = datetime.strptime(time_range.end, time_range.str_format)
+        start_adj = datetime(
+            year=start.year,
+            month=start.month,
+            day=start.day,
+            hour=start.hour,
+            minute=start.minute,
+            second=start.second,
+            microsecond=start.microsecond,
+        )
+        end_adj = datetime(
+            year=end.year,
+            month=end.month,
+            day=end.day,
+            hour=end.hour,
+            minute=end.minute,
+            second=end.second,
+            microsecond=end.microsecond,
+        )
+        freq = pd.Timedelta(time_range.frequency)
+        ranges.append((pd.Timestamp(start_adj, tz=tz), pd.Timestamp(end_adj, tz=tz), freq))
+
+    ranges.sort(key=lambda x: x[0])
+    return ranges
+
+
+def get_time_ranges(
+    time_dimension_config,  #: DateTimeDimensionConfig,
+    time_zone: str = None,
+    time_based_data_adjustment: TimeBasedDataAdjustmentModel = None,
+):
+    dim_model = time_dimension_config.model
+    if time_zone is None:
+        time_zone = time_dimension_config.get_tzinfo()
+
+    if dim_model.time_type == TimeDimensionType.DATETIME:
+        dt_ranges = dim_model.ranges
+    elif dim_model.time_type == TimeDimensionType.INDEX:
+        dt_ranges = time_dimension_config._create_represented_time_ranges()
+    else:
+        msg = f"Cannot support time_dimension_config model of time_type {dim_model.time_type}."
+        raise ValueError(msg)
+
+    ranges = []
+    for start, end, freq in build_time_ranges(dt_ranges, tz=time_zone):
+        ranges.append(
+            DatetimeRange(
+                start=start,
+                end=end,
+                frequency=freq,
+                time_based_data_adjustment=time_based_data_adjustment,
+            )
+        )
+
+    return ranges
+
+
+def is_leap_year(year: int) -> bool:
+    """Return True if the year is a leap year."""
+    return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)

dsgrid/dsgrid_rc.py

@@ -0,0 +1,88 @@
+"""Manages the dsgrid runtime configuration file"""
+
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Any
+from warnings import warn
+
+import json5
+from pydantic import model_validator
+
+from dsgrid.common import BackendEngine, DEFAULT_DB_PASSWORD, DEFAULT_SCRATCH_DIR
+from dsgrid.data_models import DSGBaseModel
+
+RC_FILENAME = ".dsgrid.json5"
+DEFAULT_BACKEND = BackendEngine.DUCKDB
+DEFAULT_THRIFT_SERVER_URL = "hive://localhost:10000/default"
+
+logger = logging.getLogger(__name__)
+
+
+class DsgridRuntimeConfig(DSGBaseModel):
+    """Defines the runtime config that can be stored in users' home directories."""
+
+    database_url: str | None = None
+    database_user: str = "root"
+    database_password: str = DEFAULT_DB_PASSWORD
+    offline: bool = True
+    backend_engine: BackendEngine = DEFAULT_BACKEND
+    thrift_server_url: str = DEFAULT_THRIFT_SERVER_URL
+    use_hive_metastore: bool = False
+    console_level: str = "info"
+    file_level: str = "info"
+    timings: bool = False
+    reraise_exceptions: bool = False
+    scratch_dir: None | Path = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def environment_overrides(cls, values: dict[str, Any]) -> dict[str, Any]:
+        for env, field in (
+            ("DSGRID_BACKEND_ENGINE", "backend_engine"),
+            ("THRIFT_SERVER_URL", "thrift_server_url"),
+        ):
+            if env in os.environ:
+                values[field] = os.environ[env]
+        return values
+
+    @model_validator(mode="before")
+    @classmethod
+    def remove_legacy_fields(cls, data: dict[str, Any]) -> dict[str, Any]:
+        for field in ("database_name",):
+            res = data.pop(field, None)
+            if res is not None:
+                warn(
+                    f"The dsgrid runtime config field {field} is deprecated. Please remove it. "
+                    "This will cause an error in a future release.",
+                )
+        return data
+
+    @classmethod
+    def load(cls) -> "DsgridRuntimeConfig":
+        """Load the dsgrid runtime config if it exists or one with default values."""
+        rc_file = cls.path()
+        if rc_file.exists():
+            data = json5.loads(rc_file.read_text(encoding="utf-8-sig"))
+            return cls(**data)
+        return cls()
+
+    def dump(self) -> None:
+        """Dump the config to the user's home directory."""
+        path = self.path()
+        data = self.model_dump()
+        data.pop("database_user")
+        data.pop("database_password")
+        with open(path, "w") as f_out:
+            json5.dump(data, f_out, indent=2)
+        print(f"Wrote dsgrid config to {path}", file=sys.stderr)
+
+    @staticmethod
+    def path() -> Path:
+        """Return the path to the config file."""
+        return Path.home() / RC_FILENAME
+
+    def get_scratch_dir(self) -> Path:
+        """Return the scratch_dir to use."""
+        return (self.scratch_dir or Path(DEFAULT_SCRATCH_DIR)).resolve()

dsgrid/exceptions.py

@@ -0,0 +1,105 @@
+"""dsgrid exceptions and warnings"""
+
+
+class DSGBaseException(Exception):
+    """Base class for all dsgrid exceptions."""
+
+
+class DSGBaseWarning(Warning):
+    """Base class for all dsgrid warnings."""
+
+
+class DSGInvalidDataset(DSGBaseException):
+    """Raised if a dataset is invalid."""
+
+
+class DSGInvalidField(DSGBaseException):
+    """Raised if a field is missing or invalid."""
+
+
+class DSGInvalidParameter(DSGBaseException):
+    """Raised if a parameter is invalid."""
+
+
+class DSGInvalidFile(DSGBaseException):
+    """Raised if a file cannot be read. Possible reason is that the write operation failed."""
+
+
+class DSGInvalidDimension(DSGBaseException):
+    """Raised if a type is not stored or is invalid."""
+
+
+class DSGInvalidDimensionAssociation(DSGBaseException):
+    """Raised if an association is not stored or is invalid."""
+
+
+class DSGInvalidDimensionMapping(DSGBaseException):
+    """Raised if a mapping is not stored or is invalid."""
+
+
+class DSGInvalidQuery(DSGBaseException):
+    """Raised if a query is invalid."""
+
+
+class DSGMissingDimensionMapping(DSGBaseException):
+    """Raised if a mapping is not provided."""
+
+
+class DSGInvalidOperation(DSGBaseException):
+    """Raised if a requested user operation is invalid."""
+
+
+class DSGRuntimeError(DSGBaseException):
+    """Raised if there was a generic runtime error."""
+
+
+class DSGProjectConfigError(DSGBaseException):
+    """Error for bad project configuration inputs"""
+
+
+class DSGDatasetConfigError(DSGBaseException):
+    """Error for bad dataset configuration inputs"""
+
+
+class DSGDuplicateValueRegistered(Warning):
+    """Raised if the user attempts to register a duplicate value."""
+
+
+class DSGValueNotRegistered(DSGBaseException):
+    """Raised if a value is not registered."""
+
+
+class DSGValueNotStored(DSGBaseException):
+    """Raised if a value is not stored."""
+
+
+class DSGConfigWarning(DSGBaseWarning):
+    """Warning for unclear or default configuration inputs"""
+
+
+class DSGFileInputError(DSGBaseException):
+    """Error during input file checks."""
+
+
+class DSGFileInputWarning(DSGBaseWarning):
+    """Warning during input file checks."""
+
+
+class DSGJSONError(DSGBaseException):
+    """Error with JSON file"""
+
+
+class DSGFilesystemInterfaceError(DSGBaseException):
+    """Error with FileSystemInterface command"""
+
+
+class DSGRegistryLockError(DSGBaseException):
+    """Error with a locked registry"""
+
+
+class DSGMakeLockError(DSGBaseException):
+    """Error when making registry lock"""
+
+
+class DSGInvalidRegistryState(DSGBaseException):
+    """Invalid Registry State"""

dsgrid/filesystem/cloud_filesystem.py

@@ -0,0 +1,32 @@
+"""Abstract implementation for a cloud filesystem"""
+
+import logging
+import abc
+
+from .filesystem_interface import FilesystemInterface
+
+logger = logging.getLogger(__name__)
+
+
+class CloudFilesystemInterface(FilesystemInterface, abc.ABC):
+    """Interface to access and edit directories and files on remote cloud filesystem"""
+
+    @abc.abstractmethod
+    def check_versions(self, directory):
+        """Check for multiple versions and versioning expectations of files.
+
+        Parameters
+        ----------
+        directory : str
+            Directory path
+        """
+
+    @abc.abstractmethod
+    def list_versions(self, path):
+        """List all versions of an S3 file object. Only possible in versioned buckets.
+
+        Parameters
+        ----------
+        path : str
+            Path
+        """