dapla-toolbelt-metadata 0.9.0__py3-none-any.whl → 0.9.2__py3-none-any.whl

dapla_metadata/datasets/__init__.py

@@ -1,6 +1,6 @@
 """Document dataset."""
 
-from datadoc_model import model
+from datadoc_model.all_optional import model
 
 from .core import Datadoc
 from .dapla_dataset_path_info import DaplaDatasetPathInfo

dapla_metadata/datasets/_merge.py

@@ -0,0 +1,333 @@
+"""Code relating to merging metadata from an existing metadata document and metadata extracted from a new dataset.
+
+This is primarily convenience functionality for users whereby they can programmatically generate metadata without
+having to manually enter it. This is primarily useful when data is sharded by time (i.e. each dataset applies for
+a particular period like a month or a year). Assuming there aren't structural changes, the metadata may be reused
+for all periods.
+
+It is important to be able to detect changes in the structure of the data and warn users about this so that they can
+make changes as appropriate.
+"""
+
+import copy
+import logging
+import warnings
+from collections.abc import Iterable
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+from typing import cast
+
+import datadoc_model
+import datadoc_model.all_optional.model as all_optional_model
+import datadoc_model.required.model as required_model
+from cloudpathlib import CloudPath
+
+from dapla_metadata.datasets.dapla_dataset_path_info import DaplaDatasetPathInfo
+from dapla_metadata.datasets.utility.constants import (
+    DATASET_FIELDS_FROM_EXISTING_METADATA,
+)
+from dapla_metadata.datasets.utility.constants import INCONSISTENCIES_MESSAGE
+from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
+from dapla_metadata.datasets.utility.utils import VariableListType
+
+logger = logging.getLogger(__name__)
+
+BUCKET_NAME_MESSAGE = "Bucket name"
+DATA_PRODUCT_NAME_MESSAGE = "Data product name"
+DATASET_STATE_MESSAGE = "Dataset state"
+DATASET_SHORT_NAME_MESSAGE = "Dataset short name"
+VARIABLES_ADDITIONAL_MESSAGE = (
+    "Dataset has additional variables than defined in metadata"
+)
+VARIABLE_RENAME_MESSAGE = "Variables have been renamed in the dataset"
+VARIABLE_ORDER_MESSAGE = "The order of variables in the dataset has changed"
+VARIABLE_DATATYPES_MESSAGE = "Variable datatypes differ"
+VARIABLES_FEWER_MESSAGE = "Dataset has fewer variables than defined in metadata"
+
+
+class InconsistentDatasetsWarning(UserWarning):
+    """Existing and new datasets differ significantly from one another."""
+
+
+class InconsistentDatasetsError(ValueError):
+    """Existing and new datasets differ significantly from one another."""
+
+
+@dataclass
+class DatasetConsistencyStatus:
+    """Store the status for different aspects of dataset consistency.
+
+    Attributes:
+        message: Communicates to the user what aspect is inconsistent.
+        success: False if inconsistency is detected.
+        variables: Optionally communicate which variables are affected.
+    """
+
+    message: str
+    success: bool
+    variables: Iterable[str] = field(default_factory=list)
+
+    def __str__(self) -> str:
+        """Format the user message."""
+        message = self.message
+        if self.variables:
+            message += f"\n\tVariables: {self.variables}"
+        return message
+
+
+def check_dataset_consistency(
+    new_dataset_path: Path | CloudPath,
+    existing_dataset_path: Path,
+) -> list[DatasetConsistencyStatus]:
+    """Run consistency tests.
+
+    Args:
+        new_dataset_path: Path to the dataset to be documented.
+        existing_dataset_path: Path stored in the existing metadata.
+
+    Returns:
+        List of consistency check results.
+    """
+    new_dataset_path_info = DaplaDatasetPathInfo(new_dataset_path)
+    existing_dataset_path_info = DaplaDatasetPathInfo(existing_dataset_path)
+    return [
+        DatasetConsistencyStatus(
+            message=BUCKET_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.bucket_name
+                == existing_dataset_path_info.bucket_name
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATA_PRODUCT_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.statistic_short_name
+                == existing_dataset_path_info.statistic_short_name
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATASET_STATE_MESSAGE,
+            success=(
+                new_dataset_path_info.dataset_state
+                == existing_dataset_path_info.dataset_state
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATASET_SHORT_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.dataset_short_name
+                == existing_dataset_path_info.dataset_short_name
+            ),
+        ),
+    ]
+
+
+def check_variables_consistency(
+    extracted_variables: VariableListType,
+    existing_variables: VariableListType,
+) -> list[DatasetConsistencyStatus]:
+    """Check for consistency in variables structure.
+
+    Compares the existing metadata and that extracted from the new dataset and provides
+    highly detailed feedback on what is different between them.
+
+    We don't return all the results because that could create conflicting messages and false positives.
+
+    Args:
+        extracted_variables (VariableListType): Variables extracted from the new dataset.
+        existing_variables (VariableListType): Variables already documented in existing metadata
+
+    Returns:
+        list[DatasetConsistencyStatus]: The list of checks and whether they were successful.
+    """
+    extracted_names_set = {v.short_name or "" for v in extracted_variables}
+    existing_names_set = {v.short_name or "" for v in existing_variables}
+    same_length = len(extracted_variables) == len(existing_variables)
+    more_extracted_variables = extracted_names_set.difference(existing_names_set)
+    fewer_extracted_variables = existing_names_set.difference(extracted_names_set)
+    results = []
+    if same_length:
+        if more_extracted_variables:
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_RENAME_MESSAGE,
+                    variables=more_extracted_variables,
+                    success=not bool(more_extracted_variables),
+                )
+            )
+        else:
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_ORDER_MESSAGE,
+                    success=[v.short_name or "" for v in extracted_variables]
+                    == [v.short_name or "" for v in existing_variables],
+                )
+            )
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_DATATYPES_MESSAGE,
+                    success=[v.data_type for v in extracted_variables]
+                    == [v.data_type for v in existing_variables],
+                )
+            )
+    else:
+        results.extend(
+            [
+                DatasetConsistencyStatus(
+                    message=VARIABLES_ADDITIONAL_MESSAGE,
+                    variables=more_extracted_variables,
+                    success=not bool(more_extracted_variables),
+                ),
+                DatasetConsistencyStatus(
+                    message=VARIABLES_FEWER_MESSAGE,
+                    variables=fewer_extracted_variables,
+                    success=not bool(fewer_extracted_variables),
+                ),
+            ]
+        )
+    return results
+
+
+def check_ready_to_merge(
+    results: list[DatasetConsistencyStatus], *, errors_as_warnings: bool
+) -> None:
+    """Check if the datasets are consistent enough to make a successful merge of metadata.
+
+    Args:
+        results: List if dict with property name and boolean success flag
+        errors_as_warnings: True if failing checks should be raised as warnings, not errors.
+
+    Raises:
+        InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+    """
+    if failures := [result for result in results if not result.success]:
+        messages_list = "\n - ".join(str(f) for f in failures)
+        msg = f"{INCONSISTENCIES_MESSAGE}\n - {messages_list}"
+        if errors_as_warnings:
+            warnings.warn(
+                message=msg,
+                category=InconsistentDatasetsWarning,
+                stacklevel=2,
+            )
+        else:
+            raise InconsistentDatasetsError(
+                msg,
+            )
+
+
+def override_dataset_fields(
+    merged_metadata: all_optional_model.DatadocMetadata,
+    existing_metadata: all_optional_model.DatadocMetadata
+    | required_model.DatadocMetadata,
+) -> None:
+    """Overrides specific fields in the dataset of `merged_metadata` with values from the dataset of `existing_metadata`.
+
+    This function iterates over a predefined list of fields, `DATASET_FIELDS_FROM_EXISTING_METADATA`,
+    and sets the corresponding fields in the `merged_metadata.dataset` object to the values
+    from the `existing_metadata.dataset` object.
+
+    Args:
+        merged_metadata: An instance of `DatadocMetadata` containing the dataset to be updated.
+        existing_metadata: An instance of `DatadocMetadata` containing the dataset whose values are used to update `merged_metadata.dataset`.
+
+    Returns:
+        `None`.
+    """
+    if merged_metadata.dataset and existing_metadata.dataset:
+        # Override the fields as defined
+        for field in DATASET_FIELDS_FROM_EXISTING_METADATA:
+            setattr(
+                merged_metadata.dataset,
+                field,
+                getattr(existing_metadata.dataset, field),
+            )
+
+
+def merge_variables(
+    existing_metadata: OptionalDatadocMetadataType,
+    extracted_metadata: all_optional_model.DatadocMetadata,
+    merged_metadata: all_optional_model.DatadocMetadata,
+) -> all_optional_model.DatadocMetadata:
+    """Merges variables from the extracted metadata into the existing metadata and updates the merged metadata.
+
+    This function compares the variables from `extracted_metadata` with those in `existing_metadata`.
+    For each variable in `extracted_metadata`, it checks if a variable with the same `short_name` exists
+    in `existing_metadata`. If a match is found, it updates the existing variable with information from
+    `extracted_metadata`. If no match is found, the variable from `extracted_metadata` is directly added to `merged_metadata`.
+
+    Args:
+        existing_metadata: The metadata object containing the current state of variables.
+        extracted_metadata: The metadata object containing new or updated variables to merge.
+        merged_metadata: The metadata object that will contain the result of the merge.
+
+    Returns:
+        all_optional_model.DatadocMetadata: The `merged_metadata` object containing variables from both `existing_metadata`
+        and `extracted_metadata`.
+    """
+    if (
+        existing_metadata is not None
+        and existing_metadata.variables is not None
+        and extracted_metadata is not None
+        and extracted_metadata.variables is not None
+        and merged_metadata.variables is not None
+    ):
+        for extracted in extracted_metadata.variables:
+            existing = next(
+                (
+                    existing
+                    for existing in existing_metadata.variables
+                    if existing.short_name == extracted.short_name
+                ),
+                None,
+            )
+            if existing:
+                existing.id = (
+                    None  # Set to None so that it will be set assigned a fresh ID later
+                )
+                existing.contains_data_from = (
+                    extracted.contains_data_from or existing.contains_data_from
+                )
+                existing.contains_data_until = (
+                    extracted.contains_data_until or existing.contains_data_until
+                )
+                merged_metadata.variables.append(
+                    cast("datadoc_model.all_optional.model.Variable", existing)
+                )
+            else:
+                # If there is no existing metadata for this variable, we just use what we have extracted
+                merged_metadata.variables.append(extracted)
+    return merged_metadata
+
+
+def merge_metadata(
+    extracted_metadata: all_optional_model.DatadocMetadata | None,
+    existing_metadata: OptionalDatadocMetadataType,
+) -> all_optional_model.DatadocMetadata:
+    if not existing_metadata:
+        logger.warning(
+            "No existing metadata found, no merge to perform. Continuing with extracted metadata.",
+        )
+        return extracted_metadata or all_optional_model.DatadocMetadata()
+
+    if not extracted_metadata:
+        return cast("all_optional_model.DatadocMetadata", existing_metadata)
+
+    # Use the extracted metadata as a base
+    merged_metadata = all_optional_model.DatadocMetadata(
+        dataset=copy.deepcopy(extracted_metadata.dataset),
+        variables=[],
+    )
+
+    override_dataset_fields(
+        merged_metadata=merged_metadata,
+        existing_metadata=existing_metadata,
+    )
+
+    # Merge variables.
+    # For each extracted variable, copy existing metadata into the merged metadata
+    return merge_variables(
+        existing_metadata=existing_metadata,
+        extracted_metadata=extracted_metadata,
+        merged_metadata=merged_metadata,
+    )

dapla_metadata/datasets/core.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 import copy
 import json
 import logging
-import warnings
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -17,6 +16,11 @@ from datadoc_model.all_optional.model import DataSetStatus
 
 from dapla_metadata._shared import config
 from dapla_metadata.dapla import user_info
+from dapla_metadata.datasets._merge import DatasetConsistencyStatus
+from dapla_metadata.datasets._merge import check_dataset_consistency
+from dapla_metadata.datasets._merge import check_ready_to_merge
+from dapla_metadata.datasets._merge import check_variables_consistency
+from dapla_metadata.datasets._merge import merge_metadata
 from dapla_metadata.datasets.compatibility import is_metadata_in_container_structure
 from dapla_metadata.datasets.compatibility import upgrade_metadata
 from dapla_metadata.datasets.dapla_dataset_path_info import DaplaDatasetPathInfo
@@ -26,7 +30,6 @@ from dapla_metadata.datasets.statistic_subject_mapping import StatisticSubjectMa
 from dapla_metadata.datasets.utility.constants import (
     DEFAULT_SPATIAL_COVERAGE_DESCRIPTION,
 )
-from dapla_metadata.datasets.utility.constants import INCONSISTENCIES_MESSAGE
 from dapla_metadata.datasets.utility.constants import METADATA_DOCUMENT_FILE_SUFFIX
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
@@ -34,7 +37,6 @@ from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
 from dapla_metadata.datasets.utility.utils import calculate_percentage
 from dapla_metadata.datasets.utility.utils import derive_assessment_from_state
 from dapla_metadata.datasets.utility.utils import get_timestamp_now
-from dapla_metadata.datasets.utility.utils import merge_variables
 from dapla_metadata.datasets.utility.utils import normalize_path
 from dapla_metadata.datasets.utility.utils import (
     num_obligatory_dataset_fields_completed,
@@ -42,7 +44,6 @@ from dapla_metadata.datasets.utility.utils import (
 from dapla_metadata.datasets.utility.utils import (
     num_obligatory_variables_fields_completed,
 )
-from dapla_metadata.datasets.utility.utils import override_dataset_fields
 from dapla_metadata.datasets.utility.utils import set_dataset_owner
 from dapla_metadata.datasets.utility.utils import set_default_values_dataset
 from dapla_metadata.datasets.utility.utils import set_default_values_variables
@@ -53,18 +54,9 @@ if TYPE_CHECKING:
 
     from cloudpathlib import CloudPath
 
-
 logger = logging.getLogger(__name__)
 
 
-class InconsistentDatasetsWarning(UserWarning):
-    """Existing and new datasets differ significantly from one another."""
-
-
-class InconsistentDatasetsError(ValueError):
-    """Existing and new datasets differ significantly from one another."""
-
-
 class Datadoc:
     """Handle reading, updating and writing of metadata.
 
@@ -118,7 +110,7 @@ class Datadoc:
         self.variables: list = []
         self.variables_lookup: dict[str, all_optional_model.Variable] = {}
         self.explicitly_defined_metadata_document = False
-        self.dataset_consistency_status: list = []
+        self.dataset_consistency_status: list[DatasetConsistencyStatus] = []
         if metadata_document_path:
             self.metadata_document = normalize_path(metadata_document_path)
             self.explicitly_defined_metadata_document = True
@@ -169,20 +161,22 @@ class Datadoc:
         ):
             extracted_metadata = self._extract_metadata_from_dataset(self.dataset_path)
 
-        if extracted_metadata is not None:
-            existing_file_path = self._get_existing_file_path(extracted_metadata)
-            if (
-                self.dataset_path
-                and existing_file_path is not None
-                and extracted_metadata is not None
-                and existing_metadata is not None
-            ):
-                self.dataset_consistency_status = self._check_dataset_consistency(
-                    self.dataset_path,
-                    Path(existing_file_path),
-                    extracted_metadata,
-                    existing_metadata,
+        if (
+            self.dataset_path
+            and self.metadata_document
+            and extracted_metadata
+            and existing_metadata
+        ):
+            self.dataset_consistency_status = check_dataset_consistency(
+                self.dataset_path,
+                Path(self.metadata_document),
+            )
+            self.dataset_consistency_status.extend(
+                check_variables_consistency(
+                    extracted_metadata.variables or [],
+                    existing_metadata.variables or [],
                 )
+            )
 
         if (
             self.dataset_path
@@ -192,11 +186,11 @@ class Datadoc:
             and extracted_metadata is not None
             and existing_metadata is not None
         ):
-            self._check_ready_to_merge(
+            check_ready_to_merge(
                 self.dataset_consistency_status,
                 errors_as_warnings=self.errors_as_warnings,
             )
-            merged_metadata = self._merge_metadata(
+            merged_metadata = merge_metadata(
                 extracted_metadata,
                 existing_metadata,
             )
@@ -214,19 +208,6 @@ class Datadoc:
         set_dataset_owner(self.dataset)
         self._create_variables_lookup()
 
-    def _get_existing_file_path(
-        self,
-        extracted_metadata: all_optional_model.DatadocMetadata | None,
-    ) -> str:
-        if (
-            extracted_metadata is not None
-            and extracted_metadata.dataset is not None
-            and extracted_metadata.dataset.file_path is not None
-        ):
-            return extracted_metadata.dataset.file_path
-        msg = "Could not access existing dataset file path"
-        raise ValueError(msg)
-
     def _set_metadata(
         self,
         merged_metadata: OptionalDatadocMetadataType,
@@ -244,134 +225,6 @@ class Datadoc:
             v.short_name: v for v in self.variables if v.short_name
         }
 
-    @staticmethod
-    def _check_dataset_consistency(
-        new_dataset_path: Path | CloudPath,
-        existing_dataset_path: Path,
-        extracted_metadata: all_optional_model.DatadocMetadata,
-        existing_metadata: OptionalDatadocMetadataType,
-    ) -> list[dict[str, object]]:
-        """Run consistency tests.
-
-        Args:
-            new_dataset_path: Path to the dataset to be documented.
-            existing_dataset_path: Path stored in the existing metadata.
-            extracted_metadata: Metadata extracted from a physical dataset.
-            existing_metadata: Metadata from a previously created metadata document.
-
-        Returns:
-            List if dict with property name and boolean success flag
-        """
-        new_dataset_path_info = DaplaDatasetPathInfo(new_dataset_path)
-        existing_dataset_path_info = DaplaDatasetPathInfo(existing_dataset_path)
-        return [
-            {
-                "name": "Bucket name",
-                "success": (
-                    new_dataset_path_info.bucket_name
-                    == existing_dataset_path_info.bucket_name
-                ),
-            },
-            {
-                "name": "Data product name",
-                "success": (
-                    new_dataset_path_info.statistic_short_name
-                    == existing_dataset_path_info.statistic_short_name
-                ),
-            },
-            {
-                "name": "Dataset state",
-                "success": (
-                    new_dataset_path_info.dataset_state
-                    == existing_dataset_path_info.dataset_state
-                ),
-            },
-            {
-                "name": "Dataset short name",
-                "success": (
-                    new_dataset_path_info.dataset_short_name
-                    == existing_dataset_path_info.dataset_short_name
-                ),
-            },
-            {
-                "name": "Variable names",
-                "success": (
-                    existing_metadata is not None
-                    and {v.short_name for v in extracted_metadata.variables or []}
-                    == {v.short_name for v in existing_metadata.variables or []}
-                ),
-            },
-            {
-                "name": "Variable datatypes",
-                "success": (
-                    existing_metadata is not None
-                    and [v.data_type for v in extracted_metadata.variables or []]
-                    == [v.data_type for v in existing_metadata.variables or []]
-                ),
-            },
-        ]
-
-    @staticmethod
-    def _check_ready_to_merge(
-        results: list[dict[str, object]], *, errors_as_warnings: bool
-    ) -> None:
-        """Check if the datasets are consistent enough to make a successful merge of metadata.
-
-        Args:
-            results: List if dict with property name and boolean success flag
-            errors_as_warnings: True if failing checks should be raised as warnings, not errors.
-
-        Raises:
-            InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
-        """
-        if failures := [result for result in results if not result["success"]]:
-            msg = f"{INCONSISTENCIES_MESSAGE} {', '.join(str(f['name']) for f in failures)}"
-            if errors_as_warnings:
-                warnings.warn(
-                    message=msg,
-                    category=InconsistentDatasetsWarning,
-                    stacklevel=2,
-                )
-            else:
-                raise InconsistentDatasetsError(
-                    msg,
-                )
-
-    @staticmethod
-    def _merge_metadata(
-        extracted_metadata: all_optional_model.DatadocMetadata | None,
-        existing_metadata: OptionalDatadocMetadataType,
-    ) -> all_optional_model.DatadocMetadata:
-        if not existing_metadata:
-            logger.warning(
-                "No existing metadata found, no merge to perform. Continuing with extracted metadata.",
-            )
-            return extracted_metadata or all_optional_model.DatadocMetadata()
-
-        if not extracted_metadata:
-            return cast("all_optional_model.DatadocMetadata", existing_metadata)
-
-        # Use the extracted metadata as a base
-        merged_metadata = all_optional_model.DatadocMetadata(
-            dataset=copy.deepcopy(extracted_metadata.dataset),
-            variables=[],
-        )
-
-        override_dataset_fields(
-            merged_metadata=merged_metadata,
-            existing_metadata=cast(
-                "all_optional_model.DatadocMetadata", existing_metadata
-            ),
-        )
-
-        # Merge variables.
-        # For each extracted variable, copy existing metadata into the merged metadata
-        return merge_variables(
-            existing_metadata=existing_metadata,
-            extracted_metadata=extracted_metadata,
-            merged_metadata=merged_metadata,
-        )
-
     def _extract_metadata_from_existing_document(
         self,
         document: pathlib.Path | CloudPath,

dapla_metadata/datasets/utility/constants.py

@@ -9,7 +9,7 @@ DATE_VALIDATION_MESSAGE = f"{VALIDATION_ERROR}contains_data_from must be the sam
 
 OBLIGATORY_METADATA_WARNING = "Obligatory metadata is missing: "
 
-INCONSISTENCIES_MESSAGE = "Inconsistencies found between extracted and existing metadata. Inconsistencies are:"
+INCONSISTENCIES_MESSAGE = "Inconsistencies found between extracted and existing metadata! This usually means that the new dataset has a different structure and that the version number should be incremented.\nDetails:"
 
 OBLIGATORY_DATASET_METADATA_IDENTIFIERS: list = [
     "assessment",
@@ -17,12 +17,9 @@ OBLIGATORY_DATASET_METADATA_IDENTIFIERS: list = [
     "dataset_status",
     "name",
     "description",
-    "data_source",
     "population_description",
     "version",
     "version_description",
-    "unit_type",
-    "temporality_type",
     "subject_field",
     "spatial_coverage_description",
     "owner",
@@ -44,6 +41,10 @@ OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS = [
     "data_type",
     "variable_role",
     "is_personal_data",
+    "unit_type",
+    "population_description",
+    "data_source",
+    "temporality_type",
 ]
 
 OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS_MULTILANGUAGE = [

dapla_metadata/datasets/utility/utils.py

@@ -5,9 +5,7 @@ import logging
 import pathlib
 import uuid
 from typing import TypeAlias
-from typing import cast
 
-import datadoc_model
 import datadoc_model.all_optional.model as all_optional_model
 import datadoc_model.required.model as required_model
 import google.auth
@@ -20,9 +18,6 @@ from datadoc_model.all_optional.model import DataSetState
 from datadoc_model.all_optional.model import VariableRole
 
 from dapla_metadata.dapla import user_info
-from dapla_metadata.datasets.utility.constants import (
-    DATASET_FIELDS_FROM_EXISTING_METADATA,
-)
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
 from dapla_metadata.datasets.utility.constants import (
     OBLIGATORY_DATASET_METADATA_IDENTIFIERS,
@@ -43,6 +38,10 @@ DatadocMetadataType: TypeAlias = (
     all_optional_model.DatadocMetadata | required_model.DatadocMetadata
 )
 DatasetType: TypeAlias = all_optional_model.Dataset | required_model.Dataset
+VariableType: TypeAlias = all_optional_model.Variable | required_model.Variable
+VariableListType: TypeAlias = (
+    list[all_optional_model.Variable] | list[required_model.Variable]
+)
 OptionalDatadocMetadataType: TypeAlias = DatadocMetadataType | None
 
 
@@ -426,87 +425,3 @@ def running_in_notebook() -> bool:
         # interpreters and will throw a NameError. Therefore we're not running
         # in Jupyter.
         return False
-
-
-def override_dataset_fields(
-    merged_metadata: all_optional_model.DatadocMetadata,
-    existing_metadata: all_optional_model.DatadocMetadata
-    | required_model.DatadocMetadata,
-) -> None:
-    """Overrides specific fields in the dataset of `merged_metadata` with values from the dataset of `existing_metadata`.
-
-    This function iterates over a predefined list of fields, `DATASET_FIELDS_FROM_EXISTING_METADATA`,
-    and sets the corresponding fields in the `merged_metadata.dataset` object to the values
-    from the `existing_metadata.dataset` object.
-
-    Args:
-        merged_metadata: An instance of `DatadocMetadata` containing the dataset to be updated.
-        existing_metadata: An instance of `DatadocMetadata` containing the dataset whose values are used to update `merged_metadata.dataset`.
-
-    Returns:
-        `None`.
-    """
-    if merged_metadata.dataset and existing_metadata.dataset:
-        # Override the fields as defined
-        for field in DATASET_FIELDS_FROM_EXISTING_METADATA:
-            setattr(
-                merged_metadata.dataset,
-                field,
-                getattr(existing_metadata.dataset, field),
-            )
-
-
-def merge_variables(
-    existing_metadata: OptionalDatadocMetadataType,
-    extracted_metadata: all_optional_model.DatadocMetadata,
-    merged_metadata: all_optional_model.DatadocMetadata,
-) -> all_optional_model.DatadocMetadata:
-    """Merges variables from the extracted metadata into the existing metadata and updates the merged metadata.
-
-    This function compares the variables from `extracted_metadata` with those in `existing_metadata`.
-    For each variable in `extracted_metadata`, it checks if a variable with the same `short_name` exists
-    in `existing_metadata`. If a match is found, it updates the existing variable with information from
-    `extracted_metadata`. If no match is found, the variable from `extracted_metadata` is directly added to `merged_metadata`.
-
-    Args:
-        existing_metadata: The metadata object containing the current state of variables.
-        extracted_metadata: The metadata object containing new or updated variables to merge.
-        merged_metadata: The metadata object that will contain the result of the merge.
-
-    Returns:
-        all_optional_model.DatadocMetadata: The `merged_metadata` object containing variables from both `existing_metadata`
-        and `extracted_metadata`.
-    """
-    if (
-        existing_metadata is not None
-        and existing_metadata.variables is not None
-        and extracted_metadata is not None
-        and extracted_metadata.variables is not None
-        and merged_metadata.variables is not None
-    ):
-        for extracted in extracted_metadata.variables:
-            existing = next(
-                (
-                    existing
-                    for existing in existing_metadata.variables
-                    if existing.short_name == extracted.short_name
-                ),
-                None,
-            )
-            if existing:
-                existing.id = (
-                    None  # Set to None so that it will be set assigned a fresh ID later
-                )
-                existing.contains_data_from = (
-                    extracted.contains_data_from or existing.contains_data_from
-                )
-                existing.contains_data_until = (
-                    extracted.contains_data_until or existing.contains_data_until
-                )
-                merged_metadata.variables.append(
-                    cast("datadoc_model.all_optional.model.Variable", existing)
-                )
-            else:
-                # If there is no existing metadata for this variable, we just use what we have extracted
-                merged_metadata.variables.append(extracted)
-    return merged_metadata

dapla_metadata/variable_definitions/__init__.py

@@ -2,6 +2,8 @@
 
 from ._generated.vardef_client import models
 from ._generated.vardef_client.exceptions import *  # noqa: F403
+from ._utils.constants import DEFAULT_DATE
+from ._utils.constants import GENERATED_CONTACT
 from .exceptions import VardefClientError
 from .exceptions import VardefFileError
 from .exceptions import VariableNotFoundError

dapla_metadata/variable_definitions/_utils/constants.py

@@ -2,6 +2,13 @@
 
 from datetime import date
 
+from dapla_metadata.variable_definitions._generated.vardef_client.models.contact import (
+    Contact,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.language_string_type import (
+    LanguageStringType,
+)
+
 VARIABLE_DEFINITIONS_DIR = "variable_definitions"
 
 VARIABLE_STATUS_FIELD_NAME = "variable_status"
@@ -26,6 +33,13 @@ TEMPLATE_SECTION_HEADER_MACHINE_GENERATED_EN = (
 
 DEFAULT_DATE = date(1000, 1, 1)
 
+GENERATED_CONTACT = Contact(
+    title=LanguageStringType(
+        nb="generert tittel",
+    ),
+    email="generert@ssb.no",
+)
+
 MACHINE_GENERATED_FIELDS = [
     "id",
     "patch_id",

dapla_metadata/variable_definitions/_utils/template_files.py

@@ -4,9 +4,6 @@ from typing import TYPE_CHECKING
 from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
     CompleteResponse,
 )
-from dapla_metadata.variable_definitions._generated.vardef_client.models.contact import (
-    Contact,
-)
 from dapla_metadata.variable_definitions._generated.vardef_client.models.language_string_type import (
     LanguageStringType,
 )
@@ -17,6 +14,7 @@ from dapla_metadata.variable_definitions._generated.vardef_client.models.variabl
     VariableStatus,
 )
 from dapla_metadata.variable_definitions._utils.constants import DEFAULT_DATE
+from dapla_metadata.variable_definitions._utils.constants import GENERATED_CONTACT
 from dapla_metadata.variable_definitions._utils.constants import TEMPLATE_HEADER
 from dapla_metadata.variable_definitions._utils.files import _create_file_name
 from dapla_metadata.variable_definitions._utils.files import _get_current_time
@@ -35,6 +33,7 @@ if TYPE_CHECKING:
 
 
 def _get_default_template() -> "VariableDefinition":
+    # Import is needed here to avoid circular imports
     from dapla_metadata.variable_definitions.variable_definition import (
         VariableDefinition,
     )
@@ -52,12 +51,7 @@ def _get_default_template() -> "VariableDefinition":
         subject_fields=[""],
         contains_special_categories_of_personal_data=False,
         owner=Owner(team="default team", groups=["default group"]),
-        contact=Contact(
-            title=LanguageStringType(
-                nb="generert tittel",
-            ),
-            email="generert@ssb.no",
-        ),
+        contact=GENERATED_CONTACT,
         variable_status=VariableStatus.DRAFT.value,
         id="",
         patch_id=0,

{dapla_toolbelt_metadata-0.9.0.dist-info → dapla_toolbelt_metadata-0.9.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dapla-toolbelt-metadata
-Version: 0.9.0
+Version: 0.9.2
 Summary: Dapla Toolbelt Metadata
 Project-URL: homepage, https://github.com/statisticsnorway/dapla-toolbelt-metadata
 Project-URL: repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata

{dapla_toolbelt_metadata-0.9.0.dist-info → dapla_toolbelt_metadata-0.9.2.dist-info}/RECORD

@@ -5,9 +5,10 @@ dapla_metadata/_shared/enums.py,sha256=WHkH1d8xw41gOly6au_izZB1_-6XTcKu5rhBWUImj
 dapla_metadata/_shared/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dapla_metadata/dapla/__init__.py,sha256=tkapF-YwmruPPrKvN3pEoCZqb7xvJx_ogBM8XyGMuJI,130
 dapla_metadata/dapla/user_info.py,sha256=bENez-ICt9ySR8orYebO68Q3_2LkIW9QTL58DTctmEQ,4833
-dapla_metadata/datasets/__init__.py,sha256=TvzskpdFC6hGcC9_55URT5jr5wNAPzXuISd2UjJWM_8,280
+dapla_metadata/datasets/__init__.py,sha256=an-REJgi7N8-S1SCz-MYO_8as6fMe03WvhjRP_hWWkg,293
+dapla_metadata/datasets/_merge.py,sha256=Tk5wQz6xZGr8veUAHZb42O8HARU8ObBJ_E4afvVWdlo,12993
 dapla_metadata/datasets/code_list.py,sha256=JtCE-5Q8grAKvkn0KKjzeGhO-96O7yGsastbuoakreg,9057
-dapla_metadata/datasets/core.py,sha256=jwnV6kqS7GpS_9kVWbFz7J0TE-TiSSiqq_gSV4sE628,25774
+dapla_metadata/datasets/core.py,sha256=r_lWZPB4zETQGXyKLwr1cgDQkeIIK_id5jGMCPHHYqg,20209
 dapla_metadata/datasets/dapla_dataset_path_info.py,sha256=WPeV_mwKk2B9sXd14SaP-kTb1bOQ_8W2KtrqOG7sJIY,26867
 dapla_metadata/datasets/dataset_parser.py,sha256=3dtRXNy1C8SfG8zTYWdY26nV4l-dG25IC_0J5t2bYwI,8285
 dapla_metadata/datasets/model_validation.py,sha256=pGT-jqaQQY4z7jz-7UQd0BQoTWDxDWPYAnDoRC2vd_c,6818
@@ -20,15 +21,15 @@ dapla_metadata/datasets/compatibility/model_backwards_compatibility.py,sha256=W5
 dapla_metadata/datasets/external_sources/__init__.py,sha256=qvIdXwqyEmXNUCB94ZtZXRzifdW4hiXASFFPtC70f6E,83
 dapla_metadata/datasets/external_sources/external_sources.py,sha256=9eIcOIUbaodNX1w9Tj2wl4U4wUmr5kF1R0i01fKUzGs,2974
 dapla_metadata/datasets/utility/__init__.py,sha256=pp6tUcgUbo8iq9OPtFKQrTbLuI3uY7NHptwWSTpasOU,33
-dapla_metadata/datasets/utility/constants.py,sha256=f9TfBN5aJbiKBQVpu8Whc0X-EMpXv43-Yu2L4KUDA4U,2353
+dapla_metadata/datasets/utility/constants.py,sha256=Iq8_0GnBwR0Ua1tPstNPJHt5mtiSehDQsW8uLNtBv_4,2489
 dapla_metadata/datasets/utility/enums.py,sha256=SpV4xlmP1YMaJPbmX03hqRLHUOhXIk5gquTeJ8G_5OE,432
-dapla_metadata/datasets/utility/utils.py,sha256=AjPWlg_8DJpqFQ8B2MaVJrB62EZ3dilvfRfc0DImjQI,18499
+dapla_metadata/datasets/utility/utils.py,sha256=f1vRy9TUbp4mtvJtXnk0Z7YTL8LZv9arcReXEm46gCg,14735
 dapla_metadata/standards/__init__.py,sha256=n8jnMrudLuScSdfQ4UMJorc-Ptg3Y1-ilT8zAaQnM70,179
 dapla_metadata/standards/name_validator.py,sha256=6-DQE_EKVd6UjL--EXpFcZDQtusVbSFaWaUY-CfOV2c,9184
 dapla_metadata/standards/standard_validators.py,sha256=tcCiCI76wUVtMzXA2oCgdauZc0uGgUi11FKu-t7KGwQ,3767
 dapla_metadata/standards/utils/__init__.py,sha256=AiM7JcpFsAgyuCyLDYZo9kI94wvIImMDGoV2lKhS4pE,42
 dapla_metadata/standards/utils/constants.py,sha256=mhWNFnS6NMsRl0c_deIdzY7_bD_wKn_oej6rzDjgwq4,2578
-dapla_metadata/variable_definitions/__init__.py,sha256=j_Nn5mnlZ2uio9moDFLE2xpALqrYpupIZMlvwbLuEuA,391
+dapla_metadata/variable_definitions/__init__.py,sha256=z48vevGb8UuQ8mwkqCtBGoyM-Ts53vUcKo7Ag5rE_Wc,482
 dapla_metadata/variable_definitions/exceptions.py,sha256=ImB81bne-h45kX9lE5hIh80QAWkOPS52uzcOftuoouM,10118
 dapla_metadata/variable_definitions/vardef.py,sha256=WUpiKfvgFGPhMdjYSFSmdlXQKAolmRgW4-t-EocddQs,13934
 dapla_metadata/variable_definitions/vardok_id.py,sha256=8T23BUHyVQr5hovTVc2E4HVY7f7e_jdi3YL1qzMQgFw,1268
@@ -85,11 +86,11 @@ dapla_metadata/variable_definitions/_generated/vardef_client/models/variable_sta
 dapla_metadata/variable_definitions/_utils/__init__.py,sha256=qAhRLJoTBqtR3f9xRXTRhD7-5Xg0Opk1Ks5F4AUYnpA,45
 dapla_metadata/variable_definitions/_utils/_client.py,sha256=v1-9VjrdPI6-sroam5vXMPEV1dQMPsYk7KyGd48HjYw,971
 dapla_metadata/variable_definitions/_utils/config.py,sha256=cc03xlcy_pnn06_6px0taO8LopmVJ9ud75oayGge8Vk,1814
-dapla_metadata/variable_definitions/_utils/constants.py,sha256=HI_XprHnuXWd4ias5qsmps1tDcUpmaWggEpEe_ZUP0g,2068
+dapla_metadata/variable_definitions/_utils/constants.py,sha256=zr5FNVCEz6TM9PVErQ672LNm-8CATrXyY94BGhV_ZJQ,2431
 dapla_metadata/variable_definitions/_utils/files.py,sha256=JbPgPNQ7iA38juMqGEdcg5OjZZUwCb6NQtPL0AEspD0,10933
-dapla_metadata/variable_definitions/_utils/template_files.py,sha256=-PgYs4TG4vrXLQgk47pow9ZsqlZqhtO755LnEmvN4MA,3405
+dapla_metadata/variable_definitions/_utils/template_files.py,sha256=7fcc7yEHOl5JUZ698kqj4IiikXPHBi3SrAVOk4wqQtw,3308
 dapla_metadata/variable_definitions/_utils/variable_definition_files.py,sha256=sGhcSpckR9NtYGNh2oVkiCd5SI3bbJEBhc1PA2uShs0,4701
-dapla_toolbelt_metadata-0.9.0.dist-info/METADATA,sha256=kbbNm1D7RWsZqqNxPaebH4nILqnXpUvcppqLZ82RJGg,4723
-dapla_toolbelt_metadata-0.9.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dapla_toolbelt_metadata-0.9.0.dist-info/licenses/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
-dapla_toolbelt_metadata-0.9.0.dist-info/RECORD,,
+dapla_toolbelt_metadata-0.9.2.dist-info/METADATA,sha256=Rf7QxJjGzKIGtHWu9PDYwv3bKzC8YRFuEMuF-EPm0lk,4723
+dapla_toolbelt_metadata-0.9.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dapla_toolbelt_metadata-0.9.2.dist-info/licenses/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
+dapla_toolbelt_metadata-0.9.2.dist-info/RECORD,,