dsgrid-toolkit 0.3.3__cp313-cp313-win_amd64.whl

dsgrid/config/dataset_schema_handler_factory.py

@@ -0,0 +1,46 @@
+from sqlalchemy import Connection
+
+from dsgrid.config.dataset_config import DatasetConfig
+from dsgrid.dataset.models import TableFormat
+from dsgrid.dataset.dataset_schema_handler_two_table import TwoTableDatasetSchemaHandler
+from dsgrid.dataset.dataset_schema_handler_one_table import OneTableDatasetSchemaHandler
+from dsgrid.registry.data_store_interface import DataStoreInterface
+from dsgrid.registry.dimension_registry_manager import DimensionRegistryManager
+from dsgrid.registry.dimension_mapping_registry_manager import DimensionMappingRegistryManager
+from dsgrid.config.dimension_mapping_base import DimensionMappingReferenceModel
+from dsgrid.utils.scratch_dir_context import ScratchDirContext
+
+
+def make_dataset_schema_handler(
+    conn: Connection | None,
+    config: DatasetConfig,
+    dimension_mgr: DimensionRegistryManager,
+    dimension_mapping_mgr: DimensionMappingRegistryManager,
+    store: DataStoreInterface | None = None,
+    mapping_references: list[DimensionMappingReferenceModel] | None = None,
+    scratch_dir_context: ScratchDirContext | None = None,
+):
+    match config.get_table_format():
+        case TableFormat.TWO_TABLE:
+            return TwoTableDatasetSchemaHandler.load(
+                config,
+                conn,
+                dimension_mgr,
+                dimension_mapping_mgr,
+                store=store,
+                mapping_references=mapping_references,
+                scratch_dir_context=scratch_dir_context,
+            )
+        case TableFormat.ONE_TABLE:
+            return OneTableDatasetSchemaHandler.load(
+                config,
+                conn,
+                dimension_mgr,
+                dimension_mapping_mgr,
+                store=store,
+                mapping_references=mapping_references,
+                scratch_dir_context=scratch_dir_context,
+            )
+        case _:
+            msg = f"Unsupported table format: {config.get_table_format()}"
+            raise NotImplementedError(msg)

dsgrid/config/date_time_dimension_config.py

@@ -0,0 +1,136 @@
+import logging
+from datetime import datetime, timedelta, tzinfo
+from zoneinfo import ZoneInfo
+
+import pandas as pd
+
+import chronify
+
+from dsgrid.dimension.time import TimeZoneFormat, TimeIntervalType
+from .dimensions import DateTimeDimensionModel
+from .time_dimension_base_config import TimeDimensionBaseConfig
+
+logger = logging.getLogger(__name__)
+
+
+class DateTimeDimensionConfig(TimeDimensionBaseConfig):
+    """Provides an interface to a DateTimeDimensionModel."""
+
+    @staticmethod
+    def model_class() -> DateTimeDimensionModel:
+        return DateTimeDimensionModel
+
+    def supports_chronify(self) -> bool:
+        return True
+
+    def to_chronify(self) -> chronify.DatetimeRange:
+        time_cols = self.get_load_data_time_columns()
+        assert len(self._model.ranges) == 1
+        assert len(time_cols) == 1
+        # TODO: issue #341: this is actually tied to the weather_year problem #340
+        # If there are no ranges, all of this must be dynamic.
+        # The two issues should be solved together.
+        datetime_type = self._get_datetime_type()
+
+        if datetime_type == "tz_aware_datetime_single_tz":
+            return chronify.DatetimeRange(
+                time_column=time_cols[0],
+                start=pd.Timestamp(self.get_start_times()[0]),
+                length=self.get_lengths()[0],
+                resolution=self.get_frequency(),
+                measurement_type=self._model.measurement_type,
+                interval_type=self._model.time_interval_type,
+            )
+        if datetime_type == "tz_naive_datetime_single_tz":
+            # localize to time zones, may do this outside of Chronify
+            msg = "dsgrid does not support NTZ datetime in dataframe yet"
+            raise NotImplementedError(msg)
+        if datetime_type == "tz_aware_datetime_multiple_tz":
+            return chronify.DatetimeRangeWithTZColumn(
+                time_column=time_cols[0],
+                start=pd.Timestamp(self.get_start_times()[0]),
+                length=self.get_lengths()[0],
+                resolution=self.get_frequency(),
+                time_zone_column="time_zone",
+                time_zones=self.get_time_zones(),
+                measurement_type=self._model.measurement_type,
+                interval_type=self._model.time_interval_type,
+            )
+        if datetime_type == "tz_naive_datetime_multiple_tz":
+            # localize to time zones, may do this outside of Chronify
+            msg = "dsgrid does not support NTZ datetime in dataframe yet"
+            raise NotImplementedError(msg)
+
+    def get_frequency(self) -> timedelta:
+        freqs = [trange.frequency for trange in self.model.ranges]
+        if len(set(freqs)) > 1:
+            msg = f"DateTimeDimensionConfig.get_frequency found multiple frequencies: {freqs}"
+            raise ValueError(msg)
+        return freqs[0]
+
+    def get_start_times(self) -> list[pd.Timestamp]:
+        tz = self.get_tzinfo()
+        start_times = []
+        for trange in self.model.ranges:
+            start = datetime.strptime(trange.start, trange.str_format)
+            assert start.tzinfo is None
+            start_times.append(start.replace(tzinfo=tz))
+        return start_times
+
+    def get_lengths(self) -> list[int]:
+        tz = self.get_tzinfo()
+        lengths = []
+        for trange in self.model.ranges:
+            start = datetime.strptime(trange.start, trange.str_format)
+            end = datetime.strptime(trange.end, trange.str_format)
+            assert start.tzinfo is None
+            assert end.tzinfo is None
+            start_utc = start.replace(tzinfo=tz).astimezone(tz=ZoneInfo("UTC"))
+            end_utc = end.replace(tzinfo=tz).astimezone(tz=ZoneInfo("UTC"))
+            freq = trange.frequency
+            length = (end_utc - start_utc) / freq + 1
+            assert length % 1 == 0, f"{length=} is not a whole number"
+            lengths.append(int(length))
+        return lengths
+
+    def get_load_data_time_columns(self) -> list[str]:
+        return [self.model.time_column]
+
+    def get_time_zone(self) -> str | None:
+        if self.model.time_zone_format.format_type == TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME:
+            return self.model.time_zone_format.time_zone
+        return None
+
+    def get_time_zones(self) -> list[str]:
+        if self.model.time_zone_format.format_type == TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME:
+            return [self.model.time_zone_format.time_zone]
+        if self.model.time_zone_format.format_type == TimeZoneFormat.ALIGNED_IN_CLOCK_TIME:
+            return self.model.time_zone_format.time_zones
+        return []
+
+    def get_tzinfo(self) -> tzinfo | None:
+        time_zone = self.get_time_zone()
+        if time_zone is None:
+            return None
+        return ZoneInfo(time_zone)
+
+    def get_time_interval_type(self) -> TimeIntervalType:
+        return self.model.time_interval_type
+
+    def _get_datetime_type(self) -> str:
+        """Return a string representing the datetime type for this dimension."""
+        match (self.model.time_zone_format.format_type, self.model.localize_to_time_zone):
+            case (TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME, True):
+                return "tz_aware_datetime_single_tz"
+            case (TimeZoneFormat.ALIGNED_IN_ABSOLUTE_TIME, False):
+                return "tz_naive_datetime_single_tz"
+            case (TimeZoneFormat.ALIGNED_IN_CLOCK_TIME, True):
+                return "tz_aware_datetime_multiple_tz"
+            case (TimeZoneFormat.ALIGNED_IN_CLOCK_TIME, False):
+                return "tz_naive_datetime_multiple_tz"
+            case _:
+                msg = (
+                    f"Unsupported combination of format_type {self.model.time_zone_format.format_type} "
+                    f"and localize_to_time_zone {self.model.localize_to_time_zone}"
+                )
+                raise ValueError(msg)

dsgrid/config/dimension_config.py

@@ -0,0 +1,54 @@
+import abc
+import logging
+from typing import Union
+
+from .config_base import ConfigBase, ConfigWithRecordFileBase
+from .dimensions import DimensionModel
+
+logger = logging.getLogger(__name__)
+
+
+class DimensionBaseConfigWithFiles(ConfigWithRecordFileBase, abc.ABC):
+    """Base class for dimension configs"""
+
+    @staticmethod
+    def config_filename():
+        return "dimension.json5"
+
+    @property
+    def config_id(self):
+        return self.model.dimension_id
+
+    def get_unique_ids(self) -> set[str]:
+        """Return the unique IDs in a dimension's records.
+
+        Returns
+        -------
+        set
+            set of str
+
+        """
+        return {x.id for x in self.model.records}
+
+
+class DimensionBaseConfigWithoutFiles(ConfigBase, abc.ABC):
+    """Base class for dimension configs"""
+
+    @staticmethod
+    def config_filename():
+        return "dimension.json5"
+
+    @property
+    def config_id(self):
+        return self.model.dimension_id
+
+
+class DimensionConfig(DimensionBaseConfigWithFiles):
+    """Provides an interface to a DimensionModel."""
+
+    @staticmethod
+    def model_class():
+        return DimensionModel
+
+
+DimensionBaseConfig = Union[DimensionBaseConfigWithFiles, DimensionBaseConfigWithoutFiles]

dsgrid/config/dimension_config_factory.py

@@ -0,0 +1,65 @@
+from dsgrid.dimension.time import TimeDimensionType
+from dsgrid.utils.files import load_data
+from .date_time_dimension_config import DateTimeDimensionConfig
+from .annual_time_dimension_config import AnnualTimeDimensionConfig
+from .noop_time_dimension_config import NoOpTimeDimensionConfig
+from .index_time_dimension_config import IndexTimeDimensionConfig
+from .dimension_config import DimensionConfig
+from .representative_period_time_dimension_config import RepresentativePeriodTimeDimensionConfig
+from .dimensions import (
+    DateTimeDimensionModel,
+    DimensionModel,
+    DimensionType,
+    AnnualTimeDimensionModel,
+    RepresentativePeriodTimeDimensionModel,
+    NoOpTimeDimensionModel,
+    IndexTimeDimensionModel,
+)
+
+
+def get_dimension_config(model):
+    if isinstance(model, DateTimeDimensionModel):
+        return DateTimeDimensionConfig(model)
+    if isinstance(model, AnnualTimeDimensionModel):
+        return AnnualTimeDimensionConfig(model)
+    if isinstance(model, RepresentativePeriodTimeDimensionModel):
+        return RepresentativePeriodTimeDimensionConfig(model)
+    if isinstance(model, DimensionModel):
+        config = DimensionConfig(model)
+        return config
+    if isinstance(model, NoOpTimeDimensionModel):
+        return NoOpTimeDimensionConfig(model)
+    if isinstance(model, IndexTimeDimensionModel):
+        return IndexTimeDimensionConfig(model)
+    assert False, type(model)
+
+
+def load_dimension_config(filename):
+    """Loads a dimension config file before the exact type is known.
+
+    Parameters
+    ----------
+    filename : Path
+
+    Returns
+    -------
+    DimensionBaseConfig
+
+    """
+    data = load_data(filename)
+    if data["type"] == DimensionType.TIME.value:
+        if data["time_type"] == TimeDimensionType.DATETIME.value:
+            return DateTimeDimensionConfig.load(filename)
+        elif data["time_type"] == TimeDimensionType.ANNUAL.value:
+            return AnnualTimeDimensionConfig.load(filename)
+        elif data["time_type"] == TimeDimensionType.REPRESENTATIVE_PERIOD.value:
+            return RepresentativePeriodTimeDimensionConfig.load(filename)
+        elif data["time_type"] == TimeDimensionType.NOOP.value:
+            return NoOpTimeDimensionConfig.load(filename)
+        elif data["time_type"] == TimeDimensionType.INDEX.value:
+            return IndexTimeDimensionConfig.load(filename)
+        else:
+            msg = f"time_type={data['time_type']} not supported"
+            raise ValueError(msg)
+
+    return DimensionConfig.load(filename)

dsgrid/config/dimension_mapping_base.py

@@ -0,0 +1,350 @@
+import logging
+
+
+from pydantic import Field, ValidationInfo, field_validator, model_validator
+
+from dsgrid.data_models import DSGBaseDatabaseModel, DSGBaseModel, DSGEnum, EnumValue
+from dsgrid.dimension.base_models import DimensionType
+from dsgrid.exceptions import DSGInvalidDimensionMapping
+from .dimensions import DimensionReferenceModel
+
+logger = logging.getLogger(__name__)
+
+
+class DimensionMappingType(DSGEnum):
+    """Defines the operation dsgrid will apply to the data during a mapping."""
+
+    # optional from_fraction col, FRACTION_SUM_EQ1 when grouped by from_id
+    ONE_TO_ONE = "one_to_one"  # includes rename, down-selection
+    MANY_TO_ONE_AGGREGATION = "many_to_one_aggregation"
+    MANY_TO_ONE_REASSIGNMENT = "many_to_one_reassignment"
+
+    # optional from_fraction col, no FRACTION_SUM check
+    DUPLICATION = "duplication"
+
+    # required from_fraction col, FRACTION_SUM_EQ1 when grouped by from_id
+    ONE_TO_MANY_DISAGGREGATION = "one_to_many_disaggregation"
+    MANY_TO_MANY_AGGREGATION = "many_to_many_aggregation"
+    MANY_TO_MANY_DISAGGREGATION = "many_to_many_disaggregation"
+
+    # required from_fraction col, FRACTION_SUM_EQ1 when grouped by to_id
+    MANY_TO_ONE_ASSIGNMENT = "many_to_one_assignment"
+    ONE_TO_MANY_ASSIGNMENT = "one_to_many_assignment"
+    MANY_TO_MANY_ASSIGNMENT = "many_to_many_assignment"
+
+    # required from_fraction col, no FRACTION_SUM check
+    ONE_TO_ONE_EXPLICIT_MULTIPLIERS = "one_to_one_explicit_multipliers"
+    ONE_TO_MANY_EXPLICIT_MULTIPLIERS = "one_to_many_explicit_multipliers"
+    MANY_TO_ONE_EXPLICIT_MULTIPLIERS = "many_to_one_explicit_multipliers"
+    MANY_TO_MANY_EXPLICIT_MULTIPLIERS = "many_to_many_explicit_multipliers"
+
+
+class DimensionMappingArchetype(DSGEnum):
+    """Dimension mapping archetype, used to check whether duplicates are allowed in from/to
+    dimensions and apply rules about the sum of the from_fraction column.
+    """
+
+    ONE_TO_ONE_MAP_FRACTION_SUM_FROM_ID_EQ1 = EnumValue(
+        value="one_to_one_map_fraction_sum_from_id_eq1",
+        description="One-to-one dimension mapping with sum of from_fraction = 1 when grouped by from_id",
+        allow_dup_from_records=False,
+        allow_dup_to_records=False,
+        check_fraction_sum_eq1_from_id=True,
+        check_fraction_sum_eq1_to_id=False,
+    )
+    ONE_TO_MANY_MAP_FRACTION_SUM_FROM_ID_EQ1 = EnumValue(
+        value="one_to_many_map_fraction_sum_from_id_eq1",
+        description="One-to-many dimension mapping with sum of from_fraction = 1 when grouped by from_id",
+        allow_dup_from_records=True,
+        allow_dup_to_records=False,
+        check_fraction_sum_eq1_from_id=True,
+        check_fraction_sum_eq1_to_id=False,
+    )
+    MANY_TO_ONE_MAP_FRACTION_SUM_FROM_ID_EQ1 = EnumValue(
+        value="many_to_one_map_fraction_sum_from_id_eq1",
+        description="Many-to-one dimension mapping with sum of from_fraction = 1 when grouped by from_id",
+        allow_dup_from_records=False,
+        allow_dup_to_records=True,
+        check_fraction_sum_eq1_from_id=True,
+        check_fraction_sum_eq1_to_id=False,
+    )
+    MANY_TO_MANY_MAP_FRACTION_SUM_FROM_ID_EQ1 = EnumValue(
+        value="many_to_many_map_fraction_sum_from_id_eq1",
+        description="Many-to-many dimension mapping with sum of from_fraction = 1 when grouped by from_id",
+        allow_dup_from_records=True,
+        allow_dup_to_records=True,
+        check_fraction_sum_eq1_from_id=True,
+        check_fraction_sum_eq1_to_id=False,
+    )
+
+    ONE_TO_ONE_MAP_FRACTION_SUM_TO_ID_EQ1 = EnumValue(
+        value="one_to_one_map_fraction_sum_to_id_eq1",
+        description="One-to-one dimension mapping with sum of from_fraction = 1 when grouped by to_id",
+        allow_dup_from_records=False,
+        allow_dup_to_records=False,
+        check_fraction_sum_eq1_from_id=False,
+        check_fraction_sum_eq1_to_id=True,
+    )
+    ONE_TO_MANY_MAP_FRACTION_SUM_TO_ID_EQ1 = EnumValue(
+        value="one_to_many_map_fraction_sum_to_id_eq1",
+        description="One-to-many dimension mapping with sum of from_fraction = 1 when grouped by to_id",
+        allow_dup_from_records=True,
+        allow_dup_to_records=False,
+        check_fraction_sum_eq1_from_id=False,
+        check_fraction_sum_eq1_to_id=True,
+    )
+    MANY_TO_ONE_MAP_FRACTION_SUM_TO_ID_EQ1 = EnumValue(
+        value="many_to_one_map_fraction_sum_to_id_eq1",
+        description="Many-to-one dimension mapping with sum of from_fraction = 1 when grouped by to_id",
+        allow_dup_from_records=False,
+        allow_dup_to_records=True,
+        check_fraction_sum_eq1_from_id=False,
+        check_fraction_sum_eq1_to_id=True,
+    )
+    MANY_TO_MANY_MAP_FRACTION_SUM_TO_ID_EQ1 = EnumValue(
+        value="many_to_many_map_fraction_sum_to_id_eq1",
+        description="Many-to-many dimension mapping with sum of from_fraction = 1 when grouped by to_id",
+        allow_dup_from_records=True,
+        allow_dup_to_records=True,
+        check_fraction_sum_eq1_from_id=False,
+        check_fraction_sum_eq1_to_id=True,
+    )
+
+    ONE_TO_ONE_MAP = EnumValue(
+        value="one_to_one_map",
+        description="One-to-one dimension mapping with no from_fraction sum check",
+        allow_dup_from_records=False,
+        allow_dup_to_records=False,
+        check_fraction_sum_eq1_from_id=False,
+        check_fraction_sum_eq1_to_id=False,
+    )
+    ONE_TO_MANY_MAP = EnumValue(
+        value="one_to_many_map",
+        description="One-to-many dimension mapping with no from_fraction sum check",
+        allow_dup_from_records=True,
+        allow_dup_to_records=False,
+        check_fraction_sum_eq1_from_id=False,
+        check_fraction_sum_eq1_to_id=False,
+    )
+    MANY_TO_ONE_MAP = EnumValue(
+        value="many_to_one_map",
+        description="Many-to-one dimension mapping with no from_fraction sum check",
+        allow_dup_from_records=False,
+        allow_dup_to_records=True,
+        check_fraction_sum_eq1_from_id=False,
+        check_fraction_sum_eq1_to_id=False,
+    )
+    MANY_TO_MANY_MAP = EnumValue(
+        value="many_to_many_map",
+        description="Many-to-many dimension mapping with no from_fraction sum check",
+        allow_dup_from_records=True,
+        allow_dup_to_records=True,
+        check_fraction_sum_eq1_from_id=False,
+        check_fraction_sum_eq1_to_id=False,
+    )
+
+
+class DimensionMappingBaseModel(DSGBaseDatabaseModel):
+    """Base class for mapping dimensions"""
+
+    mapping_type: DimensionMappingType = Field(
+        title="mapping_type",
+        description="Type/purpose of the dimension mapping",
+        default="many_to_one_aggregation",
+        json_schema_extra={
+            "options": DimensionMappingType.format_for_docs(),
+        },
+    )
+    archetype: DimensionMappingArchetype | None = Field(
+        default=None,
+        title="archetype",
+        description="Dimension mapping archetype, determined based on mapping_type",
+        json_schema_extra={
+            "dsgrid_internal": True,
+            "options": DimensionMappingArchetype.format_for_docs(),
+        },
+    )
+    from_dimension: DimensionReferenceModel = Field(
+        title="from_dimension",
+        description="From dimension",
+    )
+    to_dimension: DimensionReferenceModel = Field(
+        title="to_dimension",
+        description="To dimension",
+    )
+    from_fraction_tolerance: float = Field(
+        title="from_fraction_tolerance",
+        description="Tolerance to apply when checking from_fraction column sums",
+        default=1e-6,
+    )
+    to_fraction_tolerance: float = Field(
+        title="to_fraction_tolerance",
+        description="Tolerance to apply when checking to_fraction column sums",
+        default=1e-6,
+    )
+    description: str | None = Field(
+        default=None,
+        title="description",
+        description="Description of dimension mapping",
+    )
+    mapping_id: str | None = Field(
+        default=None,
+        title="mapping_id",
+        description="Unique dimension mapping identifier, generated by dsgrid",
+        json_schema_extra={
+            "dsgrid_internal": True,
+            "updateable": False,
+        },
+    )
+
+    @field_validator("archetype")
+    @classmethod
+    def check_archetype(cls, archetype, info: ValidationInfo):
+        if "mapping_type" not in info.data:
+            return archetype
+
+        archetype_assignment = {
+            # optional from_fraction col, FRACTION_SUM_EQ1 when grouped by from_id
+            DimensionMappingType.ONE_TO_ONE: DimensionMappingArchetype.ONE_TO_ONE_MAP_FRACTION_SUM_FROM_ID_EQ1,
+            DimensionMappingType.MANY_TO_ONE_AGGREGATION: DimensionMappingArchetype.MANY_TO_ONE_MAP_FRACTION_SUM_FROM_ID_EQ1,
+            DimensionMappingType.MANY_TO_ONE_REASSIGNMENT: DimensionMappingArchetype.MANY_TO_ONE_MAP_FRACTION_SUM_FROM_ID_EQ1,
+            # optional from_fraction col, no FRACTION_SUM check
+            DimensionMappingType.DUPLICATION: DimensionMappingArchetype.ONE_TO_MANY_MAP,
+            # required from_fraction col, FRACTION_SUM_EQ1 when grouped by from_id
+            DimensionMappingType.ONE_TO_MANY_DISAGGREGATION: DimensionMappingArchetype.ONE_TO_MANY_MAP_FRACTION_SUM_FROM_ID_EQ1,
+            DimensionMappingType.MANY_TO_MANY_AGGREGATION: DimensionMappingArchetype.MANY_TO_MANY_MAP_FRACTION_SUM_FROM_ID_EQ1,
+            DimensionMappingType.MANY_TO_MANY_DISAGGREGATION: DimensionMappingArchetype.MANY_TO_MANY_MAP_FRACTION_SUM_FROM_ID_EQ1,
+            # required from_fraction col, FRACTION_SUM_EQ1 when grouped by to_id
+            DimensionMappingType.ONE_TO_MANY_ASSIGNMENT: DimensionMappingArchetype.ONE_TO_MANY_MAP_FRACTION_SUM_TO_ID_EQ1,
+            DimensionMappingType.MANY_TO_ONE_ASSIGNMENT: DimensionMappingArchetype.MANY_TO_MANY_MAP_FRACTION_SUM_TO_ID_EQ1,
+            DimensionMappingType.MANY_TO_MANY_ASSIGNMENT: DimensionMappingArchetype.MANY_TO_MANY_MAP_FRACTION_SUM_TO_ID_EQ1,
+            # required from_fraction col, no FRACTION_SUM check
+            DimensionMappingType.ONE_TO_ONE_EXPLICIT_MULTIPLIERS: DimensionMappingArchetype.ONE_TO_ONE_MAP,
+            DimensionMappingType.ONE_TO_MANY_EXPLICIT_MULTIPLIERS: DimensionMappingArchetype.ONE_TO_MANY_MAP,
+            DimensionMappingType.MANY_TO_ONE_EXPLICIT_MULTIPLIERS: DimensionMappingArchetype.MANY_TO_ONE_MAP,
+            DimensionMappingType.MANY_TO_MANY_EXPLICIT_MULTIPLIERS: DimensionMappingArchetype.MANY_TO_MANY_MAP,
+        }
+
+        mapping_type = info.data["mapping_type"]
+        assigned_archetype = archetype_assignment[mapping_type]
+        if archetype is None:
+            archetype = assigned_archetype
+        elif archetype != assigned_archetype:
+            msg = (
+                '"mapping_type" and "archetype" are both defined AND they DO NOT correspond to each other. '
+                "archetype can be removed from config so that it can be assigned automatically based on mapping_type. "
+                f"Otherwise, {mapping_type=} should have archetype={assigned_archetype} "
+            )
+            raise DSGInvalidDimensionMapping(msg)
+        return archetype
+
+
+class DimensionMappingPreRegisteredBaseModel(DSGBaseModel):
+    """Base class for mapping soon-to-be registered dimensions. As soon as the dimensions
+    are registered this will be converted to a DimensionMappingBaseModel and then registered.
+    """
+
+    mapping_type: DimensionMappingType = Field(
+        title="mapping_type",
+        description="Type/purpose of the dimension mapping",
+        default="many_to_one_aggregation",
+        json_schema_extra={
+            "options": DimensionMappingType.format_for_docs(),
+        },
+    )
+    archetype: DimensionMappingArchetype | None = Field(
+        default=None,
+        title="archetype",
+        description="Dimension mapping archetype, determined based on mapping_type",
+        json_schema_extra={
+            "dsgrid_internal": True,
+            "options": DimensionMappingArchetype.format_for_docs(),
+        },
+    )
+    description: str | None = Field(
+        default=None,
+        title="description",
+        description="Description of dimension mapping",
+    )
+    from_fraction_tolerance: float = Field(
+        title="from_fraction_tolerance",
+        description="Tolerance value to apply to the from_fraction column",
+        default=1e-6,
+    )
+    to_fraction_tolerance: float = Field(
+        title="to_fraction_tolerance",
+        description="Tolerance value to apply to the to_fraction column",
+        default=1e-6,
+    )
+    project_base_dimension_name: str | None = Field(
+        default=None,
+        description="Name of the base dimension for which the mapping is being registered. "
+        "This is required in cases where the project has multiple base dimensions of the same "
+        "type. If None, there must only be one base dimension of this type in the project.",
+    )
+
+
+class DimensionMappingDatasetToProjectBaseModel(DimensionMappingPreRegisteredBaseModel):
+    """Base class for mapping soon-to-be registered dimensions for a dataset. Used when
+    automatically registering mappings while submitting a dataset to a project.
+    """
+
+    dimension_type: DimensionType = Field(
+        title="dimension_type",
+        description="Dimension types that will be mapped",
+    )
+
+
+class DimensionMappingReferenceModel(DSGBaseModel):
+    """Reference to a dimension mapping stored in the registry.
+
+    The DimensionMappingReferenceModel is utilized by the project configuration (project.json5) as well as by the
+    dimension mapping reference configuration (dimension_mapping_references.json5) that may be required when submitting a dataset to a project.
+    """
+
+    from_dimension_type: DimensionType = Field(
+        title="from_dimension_type",
+        description="Dimension Type",
+        json_schema_extra={
+            "options": DimensionType.format_for_docs(),
+        },
+    )
+    to_dimension_type: DimensionType = Field(
+        title="to_dimension_type",
+        description="Dimension Type",
+        json_schema_extra={
+            "options": DimensionType.format_for_docs(),
+        },
+    )
+    mapping_id: str = Field(
+        title="mapping_id",
+        description="Unique ID of the dimension mapping",
+        json_schema_extra={
+            "updateable": False,
+        },
+    )
+    version: str = Field(
+        title="version",
+        description="Version of the dimension",
+    )
+
+    # This function can be deleted once all dataset repositories have been updated.
+    @model_validator(mode="before")
+    @classmethod
+    def handle_legacy_fields(cls, values):
+        if "required_for_validation" in values:
+            logger.warning(
+                "Removing deprecated required_for_validation field from a dimension mapping reference."
+            )
+            values.pop("required_for_validation")
+
+        return values
+
+
+class DimensionMappingReferenceListModel(DSGBaseModel):
+    """List of dimension mapping references used by the dimensions_mappings.json5 config"""
+
+    references: list[DimensionMappingReferenceModel] = Field(
+        title="references",
+        description="List of dimension mapping references",
+    )