dsgrid-toolkit 0.3.3__cp313-cp313-win_amd64.whl

dsgrid/config/dimensions.py

@@ -0,0 +1,1025 @@
+import abc
+import csv
+import importlib
+import logging
+import os
+from datetime import datetime, timedelta
+from typing import Any, Union, Literal
+import copy
+
+from pydantic import field_serializer, field_validator, model_validator, Field, ValidationInfo
+from pydantic.functional_validators import BeforeValidator
+from typing_extensions import Annotated
+
+from dsgrid.data_models import DSGBaseDatabaseModel, DSGBaseModel
+from dsgrid.dimension.base_models import DimensionType, DimensionCategory
+from dsgrid.dimension.time import (
+    TimeIntervalType,
+    MeasurementType,
+    TimeDimensionType,
+    RepresentativePeriodFormat,
+    TimeZoneFormat,
+)
+from dsgrid.time.types import DatetimeTimestampType
+from dsgrid.registry.common import REGEX_VALID_REGISTRY_NAME
+from dsgrid.utils.files import compute_file_hash
+from dsgrid.utils.utilities import convert_record_dicts_to_classes
+
+
+logger = logging.getLogger(__name__)
+
+
+class DimensionBaseModel(DSGBaseDatabaseModel):
+    """Common attributes for all dimensions"""
+
+    name: str = Field(
+        title="name",
+        description="Dimension name",
+    )
+    dimension_type: DimensionType = Field(
+        title="dimension_type",
+        alias="type",
+        description="Type of the dimension",
+        json_schema_extra={
+            "options": DimensionType.format_for_docs(),
+        },
+    )
+    dimension_id: str | None = Field(
+        default=None,
+        title="dimension_id",
+        description="Unique identifier, generated by dsgrid",
+        json_schema_extra={
+            "dsg_internal": True,
+            "updateable": False,
+        },
+    )
+    module: str = Field(
+        title="module",
+        description="Python module with the dimension class",
+        default="dsgrid.dimension.standard",
+    )
+    class_name: str = Field(
+        title="class_name",
+        description="Dimension record model class name. "
+        "The dimension class defines the expected and allowable fields (and their data types)"
+        " for the dimension records file."
+        "All dimension records must have a 'id' and 'name' field."
+        "Some dimension classes support additional fields that can be used for mapping,"
+        " querying, display, etc."
+        "dsgrid in online-mode only supports dimension classes defined in the"
+        " :mod:`dsgrid.dimension.standard` module. If dsgrid does not currently support a"
+        " dimension class that you require, please contact the dsgrid-coordination team to"
+        " request a new class feature",
+        alias="class",
+    )
+    cls: Any = Field(
+        default=None,
+        title="cls",
+        description="Dimension record model class",
+        alias="dimension_class",
+        json_schema_extra={
+            "dsgrid_internal": True,
+        },
+    )
+    description: str | None = Field(
+        default=None,
+        title="description",
+        description="A description of the dimension records that is helpful, memorable, and "
+        "identifiable",
+    )
+    id: int | None = Field(
+        default=None,
+        description="Registry database ID",
+        json_schema_extra={
+            "dsgrid_internal": True,
+        },
+    )
+
+    @field_validator("name")
+    @classmethod
+    def check_name(cls, name: str) -> str:
+        if REGEX_VALID_REGISTRY_NAME.search(name) is None:
+            msg = f"dimension name={name} does not meet the requirements"
+            raise ValueError(msg)
+        return name
+
+    @field_validator("module")
+    @classmethod
+    def check_module(cls, module) -> "DimensionBaseModel":
+        if not module.startswith("dsgrid"):
+            msg = "Only dsgrid modules are supported as a dimension module."
+            raise ValueError(msg)
+        return module
+
+    @field_validator("class_name")
+    @classmethod
+    def get_dimension_class_name(cls, class_name, info: ValidationInfo):
+        """Set class_name based on inputs."""
+        if "module" not in info.data:
+            return class_name
+
+        mod = importlib.import_module(info.data["module"])
+        if not hasattr(mod, class_name):
+            if class_name is None:
+                msg = (
+                    f'There is no class "{class_name}" in module: {mod}.'
+                    "\nIf you are using a unique dimension name, you must "
+                    "specify the dimension class."
+                )
+            else:
+                msg = f"dimension class {class_name} not in {mod}"
+            raise ValueError(msg)
+
+        return class_name
+
+    @field_validator("cls")
+    @classmethod
+    def get_dimension_class(cls, dim_class, info: ValidationInfo):
+        if "module" not in info.data or "class_name" not in info.data:
+            return dim_class
+
+        if dim_class is not None:
+            msg = f"cls={dim_class} should not be set"
+            raise ValueError(msg)
+
+        return getattr(
+            importlib.import_module(info.data["module"]),
+            info.data["class_name"],
+        )
+
+    @property
+    def label(self) -> str:
+        """Return a label for the dimension to be used in user messages."""
+        return f"{self.dimension_type} {self.name}"
+
+
+class DimensionModel(DimensionBaseModel):
+    """Defines a non-time dimension"""
+
+    filename: str | None = Field(
+        title="filename",
+        alias="file",
+        default=None,
+        description="Filename containing dimension records. Only assigned for user input and "
+        "output purposes. The registry database stores records in the dimension JSON document.",
+    )
+    file_hash: str | None = Field(
+        title="file_hash",
+        description="Hash of the contents of the file",
+        json_schema_extra={
+            "dsgrid_internal": True,
+        },
+        default=None,
+    )
+    records: list = Field(
+        title="records",
+        description="Dimension records that can either be loaded from filename at "
+        "runtime or provided directly. Example of records provided directly:\n"
+        "records: [\n"
+        "    {id: 'scenario_1', name: 'Scenario 1'},\n"
+        "    {id: 'scenario_2', name: 'Scenario 2'},\n"
+        "],",
+        default=[],
+    )
+
+    @field_validator("filename")
+    @classmethod
+    def check_file(cls, filename: str) -> str:
+        """Validate that dimension file exists and has no errors"""
+        if filename is not None:
+            if not os.path.isfile(filename):
+                msg = f"file {filename} does not exist"
+                raise ValueError(msg)
+            if filename.startswith("s3://"):
+                msg = "records must exist in the local filesystem, not on S3"
+                raise ValueError(msg)
+            if not filename.endswith(".csv"):
+                msg = f"only CSV is supported: {filename}"
+                raise ValueError(msg)
+
+        return filename
+
+    @field_validator("file_hash")
+    @classmethod
+    def compute_file_hash(cls, file_hash: str, info: ValidationInfo) -> str:
+        if info.data.get("filename") is None:
+            return file_hash
+
+        if file_hash is None:
+            file_hash = compute_file_hash(info.data["filename"])
+        return file_hash
+
+    @field_validator("records")
+    @classmethod
+    def add_records(
+        cls, records: list[dict[str, Any]], info: ValidationInfo
+    ) -> list[dict[str, Any]]:
+        """Add records from the file."""
+        dim_class = info.data.get("cls")
+        if "filename" not in info.data or dim_class is None:
+            return records
+
+        if records:
+            if isinstance(records[0], dict):
+                records = convert_record_dicts_to_classes(
+                    records, dim_class, check_duplicates=["id"]
+                )
+            return records
+
+        with open(info.data["filename"], encoding="utf-8-sig") as f_in:
+            records = convert_record_dicts_to_classes(
+                csv.DictReader(f_in), dim_class, check_duplicates=["id"]
+            )
+        return records
+
+    @field_serializer("cls", "filename")
+    def serialize_cls(self, val: str, _) -> None:
+        return None
+
+
+class TimeFormatDateTimeTZModel(DSGBaseModel):
+    """Format of timestamps in a dataset is timezone-aware datetime."""
+
+    dtype: Literal["TIMESTAMP_TZ"] = "TIMESTAMP_TZ"
+    time_column: str = Field(
+        title="time_column",
+        description="Name of the timestamp column in the dataset.",
+        default=next(iter(DatetimeTimestampType._fields)),
+    )
+
+    def get_time_columns(self) -> list[str]:
+        return [self.time_column]
+
+
+class TimeFormatDateTimeNTZModel(DSGBaseModel):
+    """Format of timestamps in a dataset is timezone-naive datetime,
+    requiring localization to time zones."""
+
+    dtype: Literal["TIMESTAMP_NTZ"] = "TIMESTAMP_NTZ"
+    time_column: str = Field(
+        title="time_column",
+        description="Name of the timestamp column in the dataset.",
+        default=next(iter(DatetimeTimestampType._fields)),
+    )
+
+    def get_time_columns(self) -> list[str]:
+        return [self.time_column]
+
+
+class TimeFormatInPartsModel(DSGBaseModel):
+    """Format of timestamps in a dataset is in parts, e.g., month-day-hour format,
+    requiring conversion to datetime."""
+
+    dtype: Literal["time_format_in_parts"] = "time_format_in_parts"
+    # TODO: we may allow more columns to be None
+    year_column: str = Field(
+        title="year_column",
+        description="Name of the year column in the dataset.",
+    )
+    month_column: str = Field(
+        title="month_column",
+        description="Name of the month column in the dataset. Value is the month in a year (1 - 12)",
+    )
+    day_column: str = Field(
+        title="day_column",
+        description="Name of the day column in the dataset. Value is the day in a month (1 - 31).",
+    )
+    hour_column: str | None = Field(
+        title="hour_column",
+        description="Name of the hour column in the dataset. Value is the hour in a day (0 - 23). "
+        "If None, the hour will be set to 0 for all rows.",
+        default=None,
+    )
+    time_zone: str | None = Field(
+        default=None,
+        title="time_zone",
+        description="IANA time zone of the timestamps. Use None for time zone-naive timestamps.",
+    )
+
+    def get_time_columns(self) -> list[str]:
+        cols = [self.year_column, self.month_column, self.day_column, self.hour_column]
+        return [col for col in cols if col is not None]
+
+
+DateTimeFormat = Annotated[
+    TimeFormatDateTimeTZModel | TimeFormatDateTimeNTZModel | TimeFormatInPartsModel,
+    Field(discriminator="dtype"),
+]
+
+
+class TimeRangeModel(DSGBaseModel):
+    """Defines a continuous range of time."""
+
+    # This uses str instead of datetime because this object doesn't have the ability
+    # to serialize/deserialize by itself.
+    # We use the DatetimeRange object during processing.
+    start: str = Field(
+        title="start",
+        description="First timestamp in the data",
+    )
+    end: str = Field(
+        title="end",
+        description="Last timestamp in the data (inclusive)",
+    )
+    str_format: str = Field(
+        title="str_format",
+        default="%Y-%m-%d %H:%M:%S",
+        description="Timestamp string format (for parsing the time ranges). "
+        "The string format is used to parse the timestamps provided in the time ranges."
+        "Cheatsheet reference: `<https://strftime.org/>`_.",
+    )
+    frequency: timedelta = Field(
+        title="frequency",
+        default=timedelta(hours=1),
+        description="Resolution of the timestamps",
+    )
+
+
+class AnnualRangeModel(DSGBaseModel):
+    """Defines a continuous range of annual time."""
+
+    start: str = Field(
+        title="start",
+        description="First year in the data",
+    )
+    end: str = Field(
+        title="end",
+        description="Last year in the data (inclusive)",
+    )
+    str_format: str = Field(
+        title="str_format",
+        default="%Y",
+        description="Timestamp string format. "
+        "The string format is used to parse the timestamps provided in the time ranges. "
+        "Cheatsheet reference: `<https://strftime.org/>`_.",
+    )
+    frequency: int = Field(
+        title="frequency",
+        default=1,
+        description="Resolution of the annual time in number of years",
+    )
+
+
+class MonthRangeModel(DSGBaseModel):
+    """Defines a continuous range of time."""
+
+    # This uses str instead of datetime because this object doesn't have the ability
+    # to serialize/deserialize by itself.
+    # We use the DatetimeRange object during processing.
+    start: int = Field(
+        title="start",
+        description="First month in the data (January is 1, December is 12)",
+    )
+    end: int = Field(
+        title="end",
+        description="Last month in the data (inclusive)",
+    )
+
+
+class IndexRangeModel(DSGBaseModel):
+    """Defines a continuous range of indices."""
+
+    start: int = Field(
+        title="start",
+        description="First of indices",
+    )
+    end: int = Field(
+        title="end",
+        description="Last of indices (inclusive)",
+    )
+    starting_timestamp: str = Field(
+        title="starting timestamp",
+        description="Timestamp the start index corresponds to.",
+    )
+    str_format: str = Field(
+        title="str_format",
+        default="%Y-%m-%d %H:%M:%S",
+        description="Timestamp string format. "
+        "The string format is used to parse the starting timestamp provided. "
+        "Cheatsheet reference: `<https://strftime.org/>`_.",
+    )
+    frequency: timedelta = Field(
+        title="frequency",
+        default=timedelta(hours=1),
+        description="Resolution of the timestamps for which the index range represents.",
+    )
+
+
+class TimeDimensionBaseModel(DimensionBaseModel, abc.ABC):
+    """Defines a base model common to all time dimensions."""
+
+    time_type: TimeDimensionType = Field(
+        title="time_type",
+        default=TimeDimensionType.DATETIME,
+        description="Type of time dimension",
+        json_schema_extra={
+            "options": TimeDimensionType.format_for_docs(),
+        },
+    )
+
+    @field_serializer("cls")
+    def serialize_cls(self, val, _):
+        return None
+
+    @abc.abstractmethod
+    def is_time_zone_required_in_geography(self):
+        """Returns True if the geography dimension records must contain a time_zone column."""
+
+
+class AlignedTimeSingleTimeZone(DSGBaseModel):
+    """For each geography, data has the same set of timestamps in absolute time.
+    Timestamps in the data must be tz-aware.
+
+    E.g., data in CA and NY both start in 2018-01-01 00:00 EST.
+    """
+
+    format_type: Literal[
+        TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME
+    ] = TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME
+    time_zone: str = Field(
+        title="time_zone",
+        description="IANA time zone of data",
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def handle_legacy_fields(cls, values):
+        if values.get("format_type") == "aligned":
+            logger.warning(
+                "Renaming legacy format_type 'aligned' to 'aligned_in_absolute_time' within the datetime config time_zone_format parameter."
+            )
+            values["format_type"] = TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME.value
+
+        if "timezone" in values:
+            logger.warning(
+                "Renaming legacy timezone field to time_zone within the aligned_in_absolute_time single time zone time_zone_format."
+            )
+            values["time_zone"] = values.pop("timezone")
+        return values
+
+
+class LocalTimeMultipleTimeZones(DSGBaseModel):
+    """For each geography, data has the same set of timestamps when interpreted as local clock time by adjusting
+    for the time zone of each geography.
+    Timestamps in the data must be tz-aware.
+
+    E.g., data in CA may start in 2018-01-01 00:00 PST while data in NY may start in 2018-01-01 00:00 EST.
+    They are aligned in clock time but not in absolute time.
+
+    """
+
+    format_type: Literal[
+        TimeZoneFormat.ALIGNED_IN_CLOCK_TIME
+    ] = TimeZoneFormat.ALIGNED_IN_CLOCK_TIME
+    time_zones: list[str] = Field(
+        title="time_zones",
+        description="List of unique IANA time zones in the dataset",
+    )
+
+
+class DateTimeDimensionModel(TimeDimensionBaseModel):
+    """Defines a time dimension where timestamps translate to datetime objects."""
+
+    column_format: DateTimeFormat = Field(
+        default=TimeFormatDateTimeTZModel(),
+        title="time_format",
+        description="Specifies the format of the timestamps in the dataset.",
+    )
+    time_zone_format: Union[AlignedTimeSingleTimeZone, LocalTimeMultipleTimeZones] = Field(
+        title="time_zone_format",
+        discriminator="format_type",
+        description="Specifies whether timestamps are aligned in absolute time or in local time when adjusted for time zone.",
+    )
+
+    measurement_type: MeasurementType = Field(
+        title="measurement_type",
+        default=MeasurementType.TOTAL,
+        description="""
+        The type of measurement represented by a value associated with a timestamp:
+            mean, min, max, measured, total
+        """,
+        json_schema_extra={
+            "options": MeasurementType.format_for_docs(),
+        },
+    )
+
+    ranges: list[TimeRangeModel] = Field(
+        title="time_ranges",
+        description="Defines the continuous ranges of datetime in the data, inclusive of start and end time.",
+    )
+    time_interval_type: TimeIntervalType = Field(
+        title="time_interval",
+        description="The range of time that the value associated with a timestamp represents, e.g., period-beginning",
+        json_schema_extra={
+            "options": TimeIntervalType.format_descriptions_for_docs(),
+        },
+    )
+    time_column: str = Field(
+        title="time_column",
+        description="Name of time column in the dataframe. It should be updated during the query process to reflect "
+        "any changes to the dataframe time column.",
+        default=next(iter(DatetimeTimestampType._fields)),
+    )
+    localize_to_time_zone: bool = Field(
+        title="localize_to_time_zone",
+        default=True,
+        description="Whether to localize timestamps to time zone(s). If True, timestamps in the dataframe must be tz-naive.",
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def handle_legacy_fields(cls, values):
+        if "leap_day_adjustment" in values:
+            if values["leap_day_adjustment"] != "none":
+                msg = f"Unknown data_schema format: {values=}"
+                raise ValueError(msg)
+            logger.warning(
+                "Dropping deprecated leap_day_adjustment field from the datetime config."
+            )
+            values.pop("leap_day_adjustment")
+
+        if "datetime_format" in values:
+            logger.warning(
+                "Moving legacy datetime_format field to new time_zone_format struct within the datetime config."
+            )
+            datetime_format = values.pop("datetime_format")
+            values["time_zone_format"] = datetime_format
+
+        if "timezone" in values:
+            logger.warning(
+                "Renaming legacy timezone field to time_zone and moving it to new time_zone_format struct within the datetime config."
+            )
+            time_zone = values.pop("timezone")
+            if "time_zone_format" in values:
+                if isinstance(values["time_zone_format"], dict):
+                    assert (
+                        values["time_zone_format"].get("format_type")
+                        == TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME.value
+                    )
+                    values["time_zone_format"]["time_zone"] = time_zone
+                elif isinstance(values["time_zone_format"], AlignedTimeSingleTimeZone):
+                    assert (
+                        values["time_zone_format"].format_type
+                        == TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME
+                    )
+                    values["time_zone_format"].time_zone = time_zone
+                elif isinstance(values["time_zone_format"], LocalTimeMultipleTimeZones):
+                    msg = "Cannot set single time_zone for LocalTimeMultipleTimeZones time_zone_format."
+                    raise ValueError(msg)
+                else:
+                    msg = f"Unexpected time_zone_format type: {values['time_zone_format']}"
+                    raise ValueError(msg)
+            else:
+                values["time_zone_format"] = {
+                    "format_type": TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME.value,
+                    "time_zone": time_zone,
+                }
+
+        if "time_zone_format" in values:
+            if isinstance(values["time_zone_format"], dict):
+                if values["time_zone_format"].get("format_type") == "aligned":
+                    logger.warning(
+                        "Renaming legacy format_type 'aligned' to 'aligned_in_absolute_time' within the datetime config."
+                    )
+                    values["time_zone_format"][
+                        "format_type"
+                    ] = TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME.value
+            elif isinstance(values["time_zone_format"], AlignedTimeSingleTimeZone):
+                # already correct
+                pass
+            elif isinstance(values["time_zone_format"], LocalTimeMultipleTimeZones):
+                # already correct
+                pass
+            else:
+                msg = f"Unexpected time_zone_format type: {values['time_zone_format']}"
+                raise ValueError(msg)
+
+        if "str_format" in values:
+            logger.warning(
+                "Moving legacy str_format field to ranges struct within the datetime config."
+            )
+            str_format = values.pop("str_format")
+            for trange in values.get("ranges", []):
+                if isinstance(trange, TimeRangeModel):
+                    trange.str_format = str_format
+                elif isinstance(trange, dict):
+                    trange["str_format"] = str_format
+                else:
+                    msg = f"Unexpected ranges type: {type(trange)}"
+                    raise ValueError(msg)
+
+        if "frequency" in values:
+            logger.warning(
+                "Moving legacy frequency field to ranges struct within the datetime config."
+            )
+            frequency = values.pop("frequency")
+            for trange in values.get("ranges", []):
+                if isinstance(trange, TimeRangeModel):
+                    trange.frequency = frequency
+                elif isinstance(trange, dict):
+                    trange["frequency"] = frequency
+                else:
+                    msg = f"Unexpected ranges type: {type(trange)}"
+                    raise ValueError(msg)
+        return values
+
+    # @model_validator(mode="after")
+    # def check_frequency(self) -> "DateTimeDimensionModel":
+    #     if self.frequency in [timedelta(days=365), timedelta(days=366)]:
+    #         raise ValueError(
+    #             f"frequency={self.frequency}, datetime config does not allow 365 or 366 days frequency, "
+    #             "use class=AnnualTime, time_type=annual to specify a year series."
+    #         )
+    #     return self
+
+    @field_validator("ranges")
+    @classmethod
+    def check_times(cls, ranges: list[TimeRangeModel]) -> list[TimeRangeModel]:
+        return _check_time_ranges(ranges)
+
+    def is_time_zone_required_in_geography(self) -> bool:
+        if self.time_zone_format.format_type == TimeZoneFormat.ALIGNED_IN_CLOCK_TIME:
+            return True
+        return False
+
+
+class AnnualTimeDimensionModel(TimeDimensionBaseModel):
+    """Defines an annual time dimension where timestamps are years.
+    Each value associated with a year represents the MEASUREMENT_TYPE over the entire year.
+    i.e., MEASUREMENT_TYPE = total means the value is the total over the year, not over the range frequency.
+    """
+
+    time_type: TimeDimensionType = Field(default=TimeDimensionType.ANNUAL)
+    measurement_type: MeasurementType = Field(
+        title="measurement_type",
+        default=MeasurementType.TOTAL,
+        description="""
+        The type of measurement represented by a value associated with an annual time:
+            e.g., total
+        """,
+        json_schema_extra={
+            "options": MeasurementType.format_for_docs(),
+        },
+    )
+
+    ranges: list[AnnualRangeModel] = Field(
+        default=[],
+        title="ranges",
+        description="Defines the contiguous ranges of annual time in the data, inclusive of start and end time.",
+    )
+
+    include_leap_day: bool = Field(
+        title="include_leap_day",
+        default=False,
+        description="Whether annual time includes leap day.",
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def handle_legacy_fields(cls, values):
+        if "str_format" in values:
+            logger.warning(
+                "Moving legacy str_format field to ranges struct within the annual time config."
+            )
+            str_format = values.pop("str_format")
+            for trange in values.get("ranges", []):
+                if isinstance(trange, AnnualRangeModel):
+                    trange.str_format = str_format
+                elif isinstance(trange, dict):
+                    trange["str_format"] = str_format
+                else:
+                    msg = f"Unexpected ranges type: {type(trange)}"
+                    raise ValueError(msg)
+
+        return values
+
+    @field_validator("ranges")
+    @classmethod
+    def check_times(cls, ranges: list[AnnualRangeModel]) -> list[AnnualRangeModel]:
+        return _check_annual_ranges(ranges)
+
+    @field_validator("measurement_type")
+    @classmethod
+    def check_measurement_type(cls, measurement_type: MeasurementType) -> MeasurementType:
+        # This restriction exists because any other measurement type would require a frequency,
+        # and that isn't part of the model definition.
+        if measurement_type != MeasurementType.TOTAL:
+            msg = f"Annual time currently only supports MeasurementType total: {measurement_type}"
+            raise ValueError(msg)
+        return measurement_type
+
+    def is_time_zone_required_in_geography(self) -> bool:
+        return False
+
+
+class RepresentativePeriodTimeDimensionModel(TimeDimensionBaseModel):
+    """Defines a representative time dimension."""
+
+    time_type: TimeDimensionType = Field(default=TimeDimensionType.REPRESENTATIVE_PERIOD)
+    measurement_type: MeasurementType = Field(
+        title="measurement_type",
+        default=MeasurementType.TOTAL,
+        description="""
+        The type of measurement represented by a value associated with a timestamp:
+            e.g., mean, total
+        """,
+        json_schema_extra={
+            "options": MeasurementType.format_for_docs(),
+        },
+    )
+    format: RepresentativePeriodFormat = Field(
+        title="format",
+        description="Format of the timestamps in the load data",
+    )
+    ranges: list[MonthRangeModel] = Field(
+        title="ranges",
+        description="Defines the continuous ranges of datetime in the data, inclusive of start and end time.",
+    )
+    time_interval_type: TimeIntervalType = Field(
+        title="time_interval",
+        description="The range of time that the value associated with a timestamp represents",
+    )
+
+    def is_time_zone_required_in_geography(self) -> bool:
+        return True
+
+
+class DatetimeExternalTimeZoneDimensionModel(TimeDimensionBaseModel):
+    """Defines a time dimension where timestamps are tz-naive and require localizing to a time zone
+    using a time zone column."""
+
+    time_zone_format: Union[AlignedTimeSingleTimeZone, LocalTimeMultipleTimeZones] = Field(
+        title="time_zone_format",
+        discriminator="format_type",
+        description="Specifies whether timestamps are aligned in absolute time or in local time when adjusted for time zone.",
+    )
+    time_type: TimeDimensionType = Field(default=TimeDimensionType.DATETIME_EXTERNAL_TZ)
+    measurement_type: MeasurementType = Field(
+        title="measurement_type",
+        default=MeasurementType.TOTAL,
+        description="""
+        The type of measurement represented by a value associated with a timestamp:
+            e.g., mean, total
+        """,
+        json_schema_extra={
+            "options": MeasurementType.format_for_docs(),
+        },
+    )
+    ranges: list[TimeRangeModel] = Field(
+        title="time_ranges",
+        description="""
+        Defines the continuous ranges of time in the data, inclusive of start and end time.
+        If the timestamps are tz-naive, they will be localized to the time zones provided in the geography dimension records.
+        """,
+    )
+    time_interval_type: TimeIntervalType = Field(
+        title="time_interval",
+        description="The range of time that the value associated with a timestamp represents, e.g., period-beginning",
+        json_schema_extra={
+            "options": TimeIntervalType.format_descriptions_for_docs(),
+        },
+    )
+
+    @field_validator("ranges")
+    @classmethod
+    def check_times(cls, ranges: list[TimeRangeModel]) -> list[TimeRangeModel]:
+        return _check_time_ranges(ranges)
+
+    def is_time_zone_required_in_geography(self) -> bool:
+        return True
+
+
+class IndexTimeDimensionModel(TimeDimensionBaseModel):
+    """Defines a time dimension where timestamps are indices and requires converting to datetime."""
+
+    time_type: TimeDimensionType = Field(default=TimeDimensionType.INDEX)
+    measurement_type: MeasurementType = Field(
+        title="measurement_type",
+        default=MeasurementType.TOTAL,
+        description="""
+        The type of measurement represented by a value associated with a timestamp:
+            e.g., mean, total
+        """,
+        json_schema_extra={
+            "options": MeasurementType.format_for_docs(),
+        },
+    )
+    ranges: list[IndexRangeModel] = Field(
+        title="ranges",
+        description="Defines the continuous ranges of indices of the data, inclusive of start and end index.",
+    )
+    time_interval_type: TimeIntervalType = Field(
+        title="time_interval",
+        description="The range of time that the value associated with a timestamp represents, e.g., period-beginning",
+        json_schema_extra={
+            "options": TimeIntervalType.format_descriptions_for_docs(),
+        },
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def handle_legacy_fields(cls, values):
+        if "starting_timestamps" in values:
+            logger.warning(
+                "Moving legacy starting_timestamps field to ranges struct within the index time config."
+            )
+            assert len(values.get("starting_timestamps", [])) == len(values.get("ranges", []))
+            for trange, st in zip(values.get("ranges", []), values.get("starting_timestamps", [])):
+                trange["starting_timestamp"] = st
+            values.pop("starting_timestamps")
+
+        if "str_format" in values:
+            logger.warning(
+                "Moving legacy str_format field to ranges struct within the index time config."
+            )
+            str_format = values.pop("str_format")
+            for trange in values.get("ranges", []):
+                trange["str_format"] = str_format
+
+        if "frequency" in values:
+            logger.warning(
+                "Moving legacy frequency field to ranges struct within the index time config."
+            )
+            frequency = values.pop("frequency")
+            for trange in values.get("ranges", []):
+                trange["frequency"] = frequency
+
+        return values
+
+    @field_validator("ranges")
+    @classmethod
+    def check_indices(cls, ranges: list[IndexRangeModel]) -> list[IndexRangeModel]:
+        return _check_index_ranges(ranges)
+
+    def is_time_zone_required_in_geography(self) -> bool:
+        return True
+
+
+class NoOpTimeDimensionModel(TimeDimensionBaseModel):
+    """Defines a NoOp time dimension."""
+
+    time_type: TimeDimensionType = TimeDimensionType.NOOP
+
+    def is_time_zone_required_in_geography(self) -> bool:
+        return False
+
+
+class DimensionReferenceModel(DSGBaseModel):
+    """Reference to a dimension stored in the registry"""
+
+    dimension_type: DimensionType = Field(
+        title="dimension_type",
+        alias="type",
+        description="Type of the dimension",
+        json_schema_extra={
+            "options": DimensionType.format_for_docs(),
+        },
+    )
+    dimension_id: str = Field(
+        title="dimension_id",
+        description="Unique ID of the dimension in the registry. "
+        "The dimension ID is generated by dsgrid when a dimension is registered. "
+        "Only alphanumerics and dashes are supported.",
+    )
+    version: str = Field(
+        title="version",
+        # TODO: add notes about warnings for outdated versions DSGRID-189 & DSGRID-148
+        description="Version of the dimension. "
+        "The version string must be in semver format (e.g., '1.0.0') and it must be "
+        " a valid/existing version in the registry.",
+    )
+
+
+def handle_dimension_union(values):
+    values = copy.deepcopy(values)
+    for i, value in enumerate(values):
+        if isinstance(value, DimensionBaseModel):
+            continue
+
+        dim_type = value.get("type")
+        if dim_type is None:
+            dim_type = value["dimension_type"]
+        # NOTE: Errors inside DimensionModel or DateTimeDimensionModel will be duplicated by Pydantic
+        if dim_type == DimensionType.TIME.value:
+            # TODO add support for DatetimeExternalTimeZoneDimensionModel
+            if value["time_type"] == TimeDimensionType.DATETIME.value:
+                values[i] = DateTimeDimensionModel(**value)
+            elif value["time_type"] == TimeDimensionType.ANNUAL.value:
+                values[i] = AnnualTimeDimensionModel(**value)
+            elif value["time_type"] == TimeDimensionType.REPRESENTATIVE_PERIOD.value:
+                values[i] = RepresentativePeriodTimeDimensionModel(**value)
+            elif value["time_type"] == TimeDimensionType.INDEX.value:
+                values[i] = IndexTimeDimensionModel(**value)
+            elif value["time_type"] == TimeDimensionType.NOOP.value:
+                values[i] = NoOpTimeDimensionModel(**value)
+            else:
+                options = [x.value for x in TimeDimensionType]
+                msg = f"{value['time_type']} not supported, valid options: {options}"
+                raise ValueError(msg)
+        else:
+            values[i] = DimensionModel(**value)
+    return values
+
+
+DimensionsListModel = Annotated[
+    list[
+        Union[
+            DimensionModel,
+            DateTimeDimensionModel,
+            AnnualTimeDimensionModel,
+            RepresentativePeriodTimeDimensionModel,
+            DatetimeExternalTimeZoneDimensionModel,
+            IndexTimeDimensionModel,
+            NoOpTimeDimensionModel,
+        ]
+    ],
+    BeforeValidator(handle_dimension_union),
+]
+
+
+def _check_time_ranges(ranges: list[TimeRangeModel]) -> list[TimeRangeModel]:
+    for trange in ranges:
+        assert isinstance(trange.frequency, timedelta)
+        if trange.frequency in [timedelta(days=365), timedelta(days=366)]:
+            msg = (
+                f"{trange.frequency=}, datetime config does not allow 365 or 366 days frequency, "
+                "use class=AnnualTime, time_type=annual to specify a year series."
+            )
+            raise ValueError(msg)
+
+        # Make sure start and end time parse.
+        start = datetime.strptime(trange.start, trange.str_format)
+        end = datetime.strptime(trange.end, trange.str_format)
+        # Make sure start and end is tz-naive.
+        if start.tzinfo is not None or end.tzinfo is not None:
+            msg = (
+                f"datetime range {trange} start and end need to be tz-naive. "
+                "Pass in the time zone info via the time_zone_format parameter"
+            )
+            raise ValueError(msg)
+        if end < start:
+            msg = f"datetime range {trange} end must not be less than start."
+            raise ValueError(msg)
+        if (end - start) % trange.frequency != timedelta(0):
+            msg = f"datetime range {trange} is inconsistent with {trange.frequency}"
+            raise ValueError(msg)
+
+    return ranges
+
+
+def _check_annual_ranges(ranges: list[AnnualRangeModel]) -> list[AnnualRangeModel]:
+    for trange in ranges:
+        # Make sure start and end time parse.
+        start = datetime.strptime(trange.start, trange.str_format)
+        end = datetime.strptime(trange.end, trange.str_format)
+        freq = trange.frequency
+        if end < start:
+            msg = f"annual time range {trange} end must not be less than start."
+            raise ValueError(msg)
+
+        assert isinstance(freq, int)
+        if (end.year - start.year) % freq != 0:
+            msg = f"annual time range start and end are inconsistent with frequency: \n{trange}"
+            raise ValueError(msg)
+    return ranges
+
+
+def _check_index_ranges(ranges: list[IndexRangeModel]):
+    for trange in ranges:
+        if trange.end < trange.start:
+            msg = f"index range {trange} end must not be less than start."
+            raise ValueError(msg)
+
+    return ranges
+
+
+class DimensionCommonModel(DSGBaseModel):
+    """Common attributes for all dimensions"""
+
+    name: str
+    dimension_type: DimensionType
+    dimension_id: str
+    class_name: str
+    description: str
+
+
+class ProjectDimensionModel(DimensionCommonModel):
+    """Common attributes for all dimensions that are assigned to a project"""
+
+    category: DimensionCategory
+
+
+def create_dimension_common_model(model) -> DimensionCommonModel:
+    """Constructs an instance of DimensionBaseModel from subclasses in order to give the API
+    one common model for all dimensions. Avoids the complexity of dealing with
+    DimensionBaseModel validators.
+    """
+    fields = set(DimensionCommonModel.model_fields)
+    data = {x: getattr(model, x) for x in type(model).model_fields if x in fields}
+    return DimensionCommonModel(**data)
+
+
+def create_project_dimension_model(model, category: DimensionCategory) -> ProjectDimensionModel:
+    data = create_dimension_common_model(model).model_dump()
+    data["category"] = category.value
+    return ProjectDimensionModel(**data)