dapla-toolbelt-metadata 0.9.1__py3-none-any.whl → 0.9.3__py3-none-any.whl

dapla_metadata/datasets/__init__.py

@@ -1,6 +1,6 @@
 """Document dataset."""
 
-from datadoc_model import model
+from datadoc_model.all_optional import model
 
 from .core import Datadoc
 from .dapla_dataset_path_info import DaplaDatasetPathInfo

dapla_metadata/datasets/_merge.py

@@ -0,0 +1,333 @@
+"""Code relating to merging metadata from an existing metadata document and metadata extracted from a new dataset.
+
+This is primarily convenience functionality for users whereby they can programmatically generate metadata without
+having to manually enter it. This is primarily useful when data is sharded by time (i.e. each dataset applies for
+a particular period like a month or a year). Assuming there aren't structural changes, the metadata may be reused
+for all periods.
+
+It is important to be able to detect changes in the structure of the data and warn users about this so that they can
+make changes as appropriate.
+"""
+
+import copy
+import logging
+import warnings
+from collections.abc import Iterable
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+from typing import cast
+
+import datadoc_model
+import datadoc_model.all_optional.model as all_optional_model
+import datadoc_model.required.model as required_model
+from cloudpathlib import CloudPath
+
+from dapla_metadata.datasets.dapla_dataset_path_info import DaplaDatasetPathInfo
+from dapla_metadata.datasets.utility.constants import (
+    DATASET_FIELDS_FROM_EXISTING_METADATA,
+)
+from dapla_metadata.datasets.utility.constants import INCONSISTENCIES_MESSAGE
+from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
+from dapla_metadata.datasets.utility.utils import VariableListType
+
+logger = logging.getLogger(__name__)
+
+BUCKET_NAME_MESSAGE = "Bucket name"
+DATA_PRODUCT_NAME_MESSAGE = "Data product name"
+DATASET_STATE_MESSAGE = "Dataset state"
+DATASET_SHORT_NAME_MESSAGE = "Dataset short name"
+VARIABLES_ADDITIONAL_MESSAGE = (
+    "Dataset has additional variables than defined in metadata"
+)
+VARIABLE_RENAME_MESSAGE = "Variables have been renamed in the dataset"
+VARIABLE_ORDER_MESSAGE = "The order of variables in the dataset has changed"
+VARIABLE_DATATYPES_MESSAGE = "Variable datatypes differ"
+VARIABLES_FEWER_MESSAGE = "Dataset has fewer variables than defined in metadata"
+
+
+class InconsistentDatasetsWarning(UserWarning):
+    """Existing and new datasets differ significantly from one another."""
+
+
+class InconsistentDatasetsError(ValueError):
+    """Existing and new datasets differ significantly from one another."""
+
+
+@dataclass
+class DatasetConsistencyStatus:
+    """Store the status for different aspects of dataset consistency.
+
+    Attributes:
+        message: Communicates to the user what aspect is inconsistent.
+        success: False if inconsistency is detected.
+        variables: Optionally communicate which variables are affected.
+    """
+
+    message: str
+    success: bool
+    variables: Iterable[str] = field(default_factory=list)
+
+    def __str__(self) -> str:
+        """Format the user message."""
+        message = self.message
+        if self.variables:
+            message += f"\n\tVariables: {self.variables}"
+        return message
+
+
+def check_dataset_consistency(
+    new_dataset_path: Path | CloudPath,
+    existing_dataset_path: Path,
+) -> list[DatasetConsistencyStatus]:
+    """Run consistency tests.
+
+    Args:
+        new_dataset_path: Path to the dataset to be documented.
+        existing_dataset_path: Path stored in the existing metadata.
+
+    Returns:
+        List of consistency check results.
+    """
+    new_dataset_path_info = DaplaDatasetPathInfo(new_dataset_path)
+    existing_dataset_path_info = DaplaDatasetPathInfo(existing_dataset_path)
+    return [
+        DatasetConsistencyStatus(
+            message=BUCKET_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.bucket_name
+                == existing_dataset_path_info.bucket_name
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATA_PRODUCT_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.statistic_short_name
+                == existing_dataset_path_info.statistic_short_name
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATASET_STATE_MESSAGE,
+            success=(
+                new_dataset_path_info.dataset_state
+                == existing_dataset_path_info.dataset_state
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATASET_SHORT_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.dataset_short_name
+                == existing_dataset_path_info.dataset_short_name
+            ),
+        ),
+    ]
+
+
+def check_variables_consistency(
+    extracted_variables: VariableListType,
+    existing_variables: VariableListType,
+) -> list[DatasetConsistencyStatus]:
+    """Check for consistency in variables structure.
+
+    Compares the existing metadata and that extracted from the new dataset and provides
+    highly detailed feedback on what is different between them.
+
+    We don't return all the results because that could create conflicting messages and false positives.
+
+    Args:
+        extracted_variables (VariableListType): Variables extracted from the new dataset.
+        existing_variables (VariableListType): Variables already documented in existing metadata
+
+    Returns:
+        list[DatasetConsistencyStatus]: The list of checks and whether they were successful.
+    """
+    extracted_names_set = {v.short_name or "" for v in extracted_variables}
+    existing_names_set = {v.short_name or "" for v in existing_variables}
+    same_length = len(extracted_variables) == len(existing_variables)
+    more_extracted_variables = extracted_names_set.difference(existing_names_set)
+    fewer_extracted_variables = existing_names_set.difference(extracted_names_set)
+    results = []
+    if same_length:
+        if more_extracted_variables:
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_RENAME_MESSAGE,
+                    variables=more_extracted_variables,
+                    success=not bool(more_extracted_variables),
+                )
+            )
+        else:
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_ORDER_MESSAGE,
+                    success=[v.short_name or "" for v in extracted_variables]
+                    == [v.short_name or "" for v in existing_variables],
+                )
+            )
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_DATATYPES_MESSAGE,
+                    success=[v.data_type for v in extracted_variables]
+                    == [v.data_type for v in existing_variables],
+                )
+            )
+    else:
+        results.extend(
+            [
+                DatasetConsistencyStatus(
+                    message=VARIABLES_ADDITIONAL_MESSAGE,
+                    variables=more_extracted_variables,
+                    success=not bool(more_extracted_variables),
+                ),
+                DatasetConsistencyStatus(
+                    message=VARIABLES_FEWER_MESSAGE,
+                    variables=fewer_extracted_variables,
+                    success=not bool(fewer_extracted_variables),
+                ),
+            ]
+        )
+    return results
+
+
+def check_ready_to_merge(
+    results: list[DatasetConsistencyStatus], *, errors_as_warnings: bool
+) -> None:
+    """Check if the datasets are consistent enough to make a successful merge of metadata.
+
+    Args:
+        results: List if dict with property name and boolean success flag
+        errors_as_warnings: True if failing checks should be raised as warnings, not errors.
+
+    Raises:
+        InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+    """
+    if failures := [result for result in results if not result.success]:
+        messages_list = "\n - ".join(str(f) for f in failures)
+        msg = f"{INCONSISTENCIES_MESSAGE}\n - {messages_list}"
+        if errors_as_warnings:
+            warnings.warn(
+                message=msg,
+                category=InconsistentDatasetsWarning,
+                stacklevel=2,
+            )
+        else:
+            raise InconsistentDatasetsError(
+                msg,
+            )
+
+
+def override_dataset_fields(
+    merged_metadata: all_optional_model.DatadocMetadata,
+    existing_metadata: all_optional_model.DatadocMetadata
+    | required_model.DatadocMetadata,
+) -> None:
+    """Overrides specific fields in the dataset of `merged_metadata` with values from the dataset of `existing_metadata`.
+
+    This function iterates over a predefined list of fields, `DATASET_FIELDS_FROM_EXISTING_METADATA`,
+    and sets the corresponding fields in the `merged_metadata.dataset` object to the values
+    from the `existing_metadata.dataset` object.
+
+    Args:
+        merged_metadata: An instance of `DatadocMetadata` containing the dataset to be updated.
+        existing_metadata: An instance of `DatadocMetadata` containing the dataset whose values are used to update `merged_metadata.dataset`.
+
+    Returns:
+        `None`.
+    """
+    if merged_metadata.dataset and existing_metadata.dataset:
+        # Override the fields as defined
+        for field in DATASET_FIELDS_FROM_EXISTING_METADATA:
+            setattr(
+                merged_metadata.dataset,
+                field,
+                getattr(existing_metadata.dataset, field),
+            )
+
+
+def merge_variables(
+    existing_metadata: OptionalDatadocMetadataType,
+    extracted_metadata: all_optional_model.DatadocMetadata,
+    merged_metadata: all_optional_model.DatadocMetadata,
+) -> all_optional_model.DatadocMetadata:
+    """Merges variables from the extracted metadata into the existing metadata and updates the merged metadata.
+
+    This function compares the variables from `extracted_metadata` with those in `existing_metadata`.
+    For each variable in `extracted_metadata`, it checks if a variable with the same `short_name` exists
+    in `existing_metadata`. If a match is found, it updates the existing variable with information from
+    `extracted_metadata`. If no match is found, the variable from `extracted_metadata` is directly added to `merged_metadata`.
+
+    Args:
+        existing_metadata: The metadata object containing the current state of variables.
+        extracted_metadata: The metadata object containing new or updated variables to merge.
+        merged_metadata: The metadata object that will contain the result of the merge.
+
+    Returns:
+        all_optional_model.DatadocMetadata: The `merged_metadata` object containing variables from both `existing_metadata`
+        and `extracted_metadata`.
+    """
+    if (
+        existing_metadata is not None
+        and existing_metadata.variables is not None
+        and extracted_metadata is not None
+        and extracted_metadata.variables is not None
+        and merged_metadata.variables is not None
+    ):
+        for extracted in extracted_metadata.variables:
+            existing = next(
+                (
+                    existing
+                    for existing in existing_metadata.variables
+                    if existing.short_name == extracted.short_name
+                ),
+                None,
+            )
+            if existing:
+                existing.id = (
+                    None  # Set to None so that it will be set assigned a fresh ID later
+                )
+                existing.contains_data_from = (
+                    extracted.contains_data_from or existing.contains_data_from
+                )
+                existing.contains_data_until = (
+                    extracted.contains_data_until or existing.contains_data_until
+                )
+                merged_metadata.variables.append(
+                    cast("datadoc_model.all_optional.model.Variable", existing)
+                )
+            else:
+                # If there is no existing metadata for this variable, we just use what we have extracted
+                merged_metadata.variables.append(extracted)
+    return merged_metadata
+
+
+def merge_metadata(
+    extracted_metadata: all_optional_model.DatadocMetadata | None,
+    existing_metadata: OptionalDatadocMetadataType,
+) -> all_optional_model.DatadocMetadata:
+    if not existing_metadata:
+        logger.warning(
+            "No existing metadata found, no merge to perform. Continuing with extracted metadata.",
+        )
+        return extracted_metadata or all_optional_model.DatadocMetadata()
+
+    if not extracted_metadata:
+        return cast("all_optional_model.DatadocMetadata", existing_metadata)
+
+    # Use the extracted metadata as a base
+    merged_metadata = all_optional_model.DatadocMetadata(
+        dataset=copy.deepcopy(extracted_metadata.dataset),
+        variables=[],
+    )
+
+    override_dataset_fields(
+        merged_metadata=merged_metadata,
+        existing_metadata=existing_metadata,
+    )
+
+    # Merge variables.
+    # For each extracted variable, copy existing metadata into the merged metadata
+    return merge_variables(
+        existing_metadata=existing_metadata,
+        extracted_metadata=extracted_metadata,
+        merged_metadata=merged_metadata,
+    )

dapla_metadata/datasets/core.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 import copy
 import json
 import logging
-import warnings
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -17,6 +16,11 @@ from datadoc_model.all_optional.model import DataSetStatus
 
 from dapla_metadata._shared import config
 from dapla_metadata.dapla import user_info
+from dapla_metadata.datasets._merge import DatasetConsistencyStatus
+from dapla_metadata.datasets._merge import check_dataset_consistency
+from dapla_metadata.datasets._merge import check_ready_to_merge
+from dapla_metadata.datasets._merge import check_variables_consistency
+from dapla_metadata.datasets._merge import merge_metadata
 from dapla_metadata.datasets.compatibility import is_metadata_in_container_structure
 from dapla_metadata.datasets.compatibility import upgrade_metadata
 from dapla_metadata.datasets.dapla_dataset_path_info import DaplaDatasetPathInfo
@@ -26,7 +30,6 @@ from dapla_metadata.datasets.statistic_subject_mapping import StatisticSubjectMa
 from dapla_metadata.datasets.utility.constants import (
     DEFAULT_SPATIAL_COVERAGE_DESCRIPTION,
 )
-from dapla_metadata.datasets.utility.constants import INCONSISTENCIES_MESSAGE
 from dapla_metadata.datasets.utility.constants import METADATA_DOCUMENT_FILE_SUFFIX
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
@@ -34,7 +37,6 @@ from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
 from dapla_metadata.datasets.utility.utils import calculate_percentage
 from dapla_metadata.datasets.utility.utils import derive_assessment_from_state
 from dapla_metadata.datasets.utility.utils import get_timestamp_now
-from dapla_metadata.datasets.utility.utils import merge_variables
 from dapla_metadata.datasets.utility.utils import normalize_path
 from dapla_metadata.datasets.utility.utils import (
     num_obligatory_dataset_fields_completed,
@@ -42,9 +44,9 @@ from dapla_metadata.datasets.utility.utils import (
 from dapla_metadata.datasets.utility.utils import (
     num_obligatory_variables_fields_completed,
 )
-from dapla_metadata.datasets.utility.utils import override_dataset_fields
 from dapla_metadata.datasets.utility.utils import set_dataset_owner
 from dapla_metadata.datasets.utility.utils import set_default_values_dataset
+from dapla_metadata.datasets.utility.utils import set_default_values_pseudonymization
 from dapla_metadata.datasets.utility.utils import set_default_values_variables
 
 if TYPE_CHECKING:
@@ -53,18 +55,9 @@ if TYPE_CHECKING:
 
     from cloudpathlib import CloudPath
 
-
 logger = logging.getLogger(__name__)
 
 
-class InconsistentDatasetsWarning(UserWarning):
-    """Existing and new datasets differ significantly from one another."""
-
-
-class InconsistentDatasetsError(ValueError):
-    """Existing and new datasets differ significantly from one another."""
-
-
 class Datadoc:
     """Handle reading, updating and writing of metadata.
 
@@ -118,7 +111,7 @@ class Datadoc:
         self.variables: list = []
         self.variables_lookup: dict[str, all_optional_model.Variable] = {}
         self.explicitly_defined_metadata_document = False
-        self.dataset_consistency_status: list = []
+        self.dataset_consistency_status: list[DatasetConsistencyStatus] = []
         if metadata_document_path:
             self.metadata_document = normalize_path(metadata_document_path)
             self.explicitly_defined_metadata_document = True
@@ -169,20 +162,22 @@ class Datadoc:
         ):
             extracted_metadata = self._extract_metadata_from_dataset(self.dataset_path)
 
-        if extracted_metadata is not None:
-            existing_file_path = self._get_existing_file_path(extracted_metadata)
-            if (
-                self.dataset_path
-                and existing_file_path is not None
-                and extracted_metadata is not None
-                and existing_metadata is not None
-            ):
-                self.dataset_consistency_status = self._check_dataset_consistency(
-                    self.dataset_path,
-                    Path(existing_file_path),
-                    extracted_metadata,
-                    existing_metadata,
+        if (
+            self.dataset_path
+            and self.metadata_document
+            and extracted_metadata
+            and existing_metadata
+        ):
+            self.dataset_consistency_status = check_dataset_consistency(
+                self.dataset_path,
+                Path(self.metadata_document),
+            )
+            self.dataset_consistency_status.extend(
+                check_variables_consistency(
+                    extracted_metadata.variables or [],
+                    existing_metadata.variables or [],
                 )
+            )
 
         if (
             self.dataset_path
@@ -192,11 +187,11 @@ class Datadoc:
             and extracted_metadata is not None
             and existing_metadata is not None
         ):
-            self._check_ready_to_merge(
+            check_ready_to_merge(
                 self.dataset_consistency_status,
                 errors_as_warnings=self.errors_as_warnings,
             )
-            merged_metadata = self._merge_metadata(
+            merged_metadata = merge_metadata(
                 extracted_metadata,
                 existing_metadata,
             )
@@ -214,19 +209,6 @@ class Datadoc:
         set_dataset_owner(self.dataset)
         self._create_variables_lookup()
 
-    def _get_existing_file_path(
-        self,
-        extracted_metadata: all_optional_model.DatadocMetadata | None,
-    ) -> str:
-        if (
-            extracted_metadata is not None
-            and extracted_metadata.dataset is not None
-            and extracted_metadata.dataset.file_path is not None
-        ):
-            return extracted_metadata.dataset.file_path
-        msg = "Could not access existing dataset file path"
-        raise ValueError(msg)
-
     def _set_metadata(
         self,
         merged_metadata: OptionalDatadocMetadataType,
@@ -244,134 +226,6 @@ class Datadoc:
             v.short_name: v for v in self.variables if v.short_name
         }
 
-    @staticmethod
-    def _check_dataset_consistency(
-        new_dataset_path: Path | CloudPath,
-        existing_dataset_path: Path,
-        extracted_metadata: all_optional_model.DatadocMetadata,
-        existing_metadata: OptionalDatadocMetadataType,
-    ) -> list[dict[str, object]]:
-        """Run consistency tests.
-
-        Args:
-            new_dataset_path: Path to the dataset to be documented.
-            existing_dataset_path: Path stored in the existing metadata.
-            extracted_metadata: Metadata extracted from a physical dataset.
-            existing_metadata: Metadata from a previously created metadata document.
-
-        Returns:
-            List if dict with property name and boolean success flag
-        """
-        new_dataset_path_info = DaplaDatasetPathInfo(new_dataset_path)
-        existing_dataset_path_info = DaplaDatasetPathInfo(existing_dataset_path)
-        return [
-            {
-                "name": "Bucket name",
-                "success": (
-                    new_dataset_path_info.bucket_name
-                    == existing_dataset_path_info.bucket_name
-                ),
-            },
-            {
-                "name": "Data product name",
-                "success": (
-                    new_dataset_path_info.statistic_short_name
-                    == existing_dataset_path_info.statistic_short_name
-                ),
-            },
-            {
-                "name": "Dataset state",
-                "success": (
-                    new_dataset_path_info.dataset_state
-                    == existing_dataset_path_info.dataset_state
-                ),
-            },
-            {
-                "name": "Dataset short name",
-                "success": (
-                    new_dataset_path_info.dataset_short_name
-                    == existing_dataset_path_info.dataset_short_name
-                ),
-            },
-            {
-                "name": "Variable names",
-                "success": (
-                    existing_metadata is not None
-                    and {v.short_name for v in extracted_metadata.variables or []}
-                    == {v.short_name for v in existing_metadata.variables or []}
-                ),
-            },
-            {
-                "name": "Variable datatypes",
-                "success": (
-                    existing_metadata is not None
-                    and [v.data_type for v in extracted_metadata.variables or []]
-                    == [v.data_type for v in existing_metadata.variables or []]
-                ),
-            },
-        ]
-
-    @staticmethod
-    def _check_ready_to_merge(
-        results: list[dict[str, object]], *, errors_as_warnings: bool
-    ) -> None:
-        """Check if the datasets are consistent enough to make a successful merge of metadata.
-
-        Args:
-            results: List if dict with property name and boolean success flag
-            errors_as_warnings: True if failing checks should be raised as warnings, not errors.
-
-        Raises:
-            InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
-        """
-        if failures := [result for result in results if not result["success"]]:
-            msg = f"{INCONSISTENCIES_MESSAGE} {', '.join(str(f['name']) for f in failures)}"
-            if errors_as_warnings:
-                warnings.warn(
-                    message=msg,
-                    category=InconsistentDatasetsWarning,
-                    stacklevel=2,
-                )
-            else:
-                raise InconsistentDatasetsError(
-                    msg,
-                )
-
-    @staticmethod
-    def _merge_metadata(
-        extracted_metadata: all_optional_model.DatadocMetadata | None,
-        existing_metadata: OptionalDatadocMetadataType,
-    ) -> all_optional_model.DatadocMetadata:
-        if not existing_metadata:
-            logger.warning(
-                "No existing metadata found, no merge to perform. Continuing with extracted metadata.",
-            )
-            return extracted_metadata or all_optional_model.DatadocMetadata()
-
-        if not extracted_metadata:
-            return cast("all_optional_model.DatadocMetadata", existing_metadata)
-
-        # Use the extracted metadata as a base
-        merged_metadata = all_optional_model.DatadocMetadata(
-            dataset=copy.deepcopy(extracted_metadata.dataset),
-            variables=[],
-        )
-
-        override_dataset_fields(
-            merged_metadata=merged_metadata,
-            existing_metadata=cast(
-                "all_optional_model.DatadocMetadata", existing_metadata
-            ),
-        )
-
-        # Merge variables.
-        # For each extracted variable, copy existing metadata into the merged metadata
-        return merge_variables(
-            existing_metadata=existing_metadata,
-            extracted_metadata=extracted_metadata,
-            merged_metadata=merged_metadata,
-        )
-
     def _extract_metadata_from_existing_document(
         self,
         document: pathlib.Path | CloudPath,
@@ -591,9 +445,12 @@ class Datadoc:
         variable_short_name: str,
         pseudonymization: all_optional_model.Pseudonymization | None = None,
     ) -> None:
-        """Adds a new pseudo variable to the list of pseudonymized variables also sets is_personal_data to true.
+        """Adds a new pseudo variable to the list of pseudonymized variables.
 
-        If there is no pseudonymization supplied an empty Pseudonymization structure will be added to the model.
+        If `pseudonymization` is not supplied, an empty Pseudonymization structure
+        will be created and assigned to the variable.
+        If an encryption algorithm is recognized (one of the standard Dapla algorithms), default values are filled
+        for any missing fields.
 
         Args:
             variable_short_name: The short name for the variable that one wants to update the pseudo for.
@@ -601,15 +458,15 @@ class Datadoc:
 
         """
         variable = self.variables_lookup[variable_short_name]
-        variable.pseudonymization = (
-            pseudonymization or all_optional_model.Pseudonymization()
-        )
+        if pseudonymization:
+            set_default_values_pseudonymization(variable, pseudonymization)
+        else:
+            variable.pseudonymization = all_optional_model.Pseudonymization()
 
     def remove_pseudonymization(self, variable_short_name: str) -> None:
         """Removes a pseudo variable by using the shortname.
 
         Updates the pseudo variable lookup by creating a new one.
-        Sets is_personal_data to non pseudonymized encrypted personal data.
 
         Args:
             variable_short_name: The short name for the variable that one wants to remove the pseudo for.

dapla_metadata/datasets/utility/constants.py

@@ -9,7 +9,7 @@ DATE_VALIDATION_MESSAGE = f"{VALIDATION_ERROR}contains_data_from must be the sam
 
 OBLIGATORY_METADATA_WARNING = "Obligatory metadata is missing: "
 
-INCONSISTENCIES_MESSAGE = "Inconsistencies found between extracted and existing metadata. Inconsistencies are:"
+INCONSISTENCIES_MESSAGE = "Inconsistencies found between extracted and existing metadata! This usually means that the new dataset has a different structure and that the version number should be incremented.\nDetails:"
 
 OBLIGATORY_DATASET_METADATA_IDENTIFIERS: list = [
     "assessment",
@@ -17,12 +17,9 @@ OBLIGATORY_DATASET_METADATA_IDENTIFIERS: list = [
     "dataset_status",
     "name",
     "description",
-    "data_source",
     "population_description",
     "version",
     "version_description",
-    "unit_type",
-    "temporality_type",
     "subject_field",
     "spatial_coverage_description",
     "owner",
@@ -44,8 +41,18 @@ OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS = [
     "data_type",
     "variable_role",
     "is_personal_data",
+    "unit_type",
+    "population_description",
+    "data_source",
+    "temporality_type",
+]
+
+OBLIGATORY_VARIABLES_PESUODONYMIZATION_IDENTIFIERS = [
+    "encryption_algorithm",
+    "encryption_key_refrence",
 ]
 
+
 OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS_MULTILANGUAGE = [
     "name",
 ]
@@ -90,3 +97,11 @@ METADATA_DOCUMENT_FILE_SUFFIX = "__DOC.json"
 DATADOC_STATISTICAL_SUBJECT_SOURCE_URL = (
     "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
 )
+
+PAPIS_STABLE_IDENTIFIER_TYPE = "FREG_SNR"
+PAPIS_ENCRYPTION_KEY_REFERENCE = "papis-common-key-1"
+DAEAD_ENCRYPTION_KEY_REFERENCE = "ssb-common-key-1"
+ENCRYPTION_PARAMETER_SNAPSHOT_DATE = "snapshotDate"
+ENCRYPTION_PARAMETER_KEY_ID = "keyId"
+ENCRYPTION_PARAMETER_STRATEGY = "strategy"
+ENCRYPTION_PARAMETER_STRATEGY_SKIP = "skip"

dapla_metadata/datasets/utility/enums.py

@@ -14,3 +14,10 @@ class SupportedLanguages(str, Enum):
     NORSK_BOKMÅL = "nb"  # noqa: PLC2401 the listed problems do not apply in this case
     NORSK_NYNORSK = "nn"
     ENGLISH = "en"
+
+
+class EncryptionAlgorithm(str, Enum):
+    """Encryption algorithm values for pseudonymization algoprithms offered on Dapla."""
+
+    PAPIS_ENCRYPTION_ALGORITHM = "TINK-FPE"
+    DAEAD_ENCRYPTION_ALGORITHM = "TINK-DAEAD"

dapla_metadata/datasets/utility/utils.py

@@ -4,25 +4,25 @@ import datetime  # import is needed in xdoctest
 import logging
 import pathlib
 import uuid
+from typing import Any
 from typing import TypeAlias
-from typing import cast
 
-import datadoc_model
 import datadoc_model.all_optional.model as all_optional_model
 import datadoc_model.required.model as required_model
 import google.auth
 from cloudpathlib import CloudPath
 from cloudpathlib import GSClient
 from cloudpathlib import GSPath
-from datadoc_model import model
 from datadoc_model.all_optional.model import Assessment
 from datadoc_model.all_optional.model import DataSetState
 from datadoc_model.all_optional.model import VariableRole
 
 from dapla_metadata.dapla import user_info
-from dapla_metadata.datasets.utility.constants import (
-    DATASET_FIELDS_FROM_EXISTING_METADATA,
-)
+from dapla_metadata.datasets.utility.constants import DAEAD_ENCRYPTION_KEY_REFERENCE
+from dapla_metadata.datasets.utility.constants import ENCRYPTION_PARAMETER_KEY_ID
+from dapla_metadata.datasets.utility.constants import ENCRYPTION_PARAMETER_SNAPSHOT_DATE
+from dapla_metadata.datasets.utility.constants import ENCRYPTION_PARAMETER_STRATEGY
+from dapla_metadata.datasets.utility.constants import ENCRYPTION_PARAMETER_STRATEGY_SKIP
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
 from dapla_metadata.datasets.utility.constants import (
     OBLIGATORY_DATASET_METADATA_IDENTIFIERS,
@@ -36,6 +36,9 @@ from dapla_metadata.datasets.utility.constants import (
 from dapla_metadata.datasets.utility.constants import (
     OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS_MULTILANGUAGE,
 )
+from dapla_metadata.datasets.utility.constants import PAPIS_ENCRYPTION_KEY_REFERENCE
+from dapla_metadata.datasets.utility.constants import PAPIS_STABLE_IDENTIFIER_TYPE
+from dapla_metadata.datasets.utility.enums import EncryptionAlgorithm
 
 logger = logging.getLogger(__name__)
 
@@ -43,9 +46,21 @@ DatadocMetadataType: TypeAlias = (
     all_optional_model.DatadocMetadata | required_model.DatadocMetadata
 )
 DatasetType: TypeAlias = all_optional_model.Dataset | required_model.Dataset
+VariableType: TypeAlias = all_optional_model.Variable | required_model.Variable
+PseudonymizationType: TypeAlias = (
+    all_optional_model.Pseudonymization | required_model.Pseudonymization
+)
+VariableListType: TypeAlias = (
+    list[all_optional_model.Variable] | list[required_model.Variable]
+)
 OptionalDatadocMetadataType: TypeAlias = DatadocMetadataType | None
 
 
+def get_current_date() -> str:
+    """Return a current date as str."""
+    return datetime.datetime.now(tz=datetime.timezone.utc).date().isoformat()
+
+
 def get_timestamp_now() -> datetime.datetime:
     """Return a timestamp for the current moment."""
     return datetime.datetime.now(tz=datetime.timezone.utc)
@@ -110,7 +125,7 @@ def set_default_values_variables(variables: list) -> None:
         variables: A list of variable objects to set default values on.
 
     Example:
-        >>> variables = [model.Variable(short_name="pers",id=None, is_personal_data = None), model.Variable(short_name="fnr",id='9662875c-c245-41de-b667-12ad2091a1ee', is_personal_data=True)]
+        >>> variables = [all_optional_model.Variable(short_name="pers",id=None, is_personal_data = None), all_optional_model.Variable(short_name="fnr",id='9662875c-c245-41de-b667-12ad2091a1ee', is_personal_data=True)]
         >>> set_default_values_variables(variables)
         >>> isinstance(variables[0].id, uuid.UUID)
         True
@@ -139,7 +154,7 @@ def set_default_values_dataset(
         dataset: The dataset object to set default values on.
 
     Example:
-        >>> dataset = model.Dataset(id=None)
+        >>> dataset = all_optional_model.Dataset(id=None)
         >>> set_default_values_dataset(dataset)
         >>> dataset.id is not None
         True
@@ -177,8 +192,8 @@ def set_variables_inherit_from_dataset(
         variables: A list of variable objects to update with dataset values.
 
     Example:
-        >>> dataset = model.Dataset(short_name='person_data_v1', id='9662875c-c245-41de-b667-12ad2091a1ee', contains_data_from="2010-09-05", contains_data_until="2022-09-05")
-        >>> variables = [model.Variable(short_name="pers", data_source=None, temporality_type=None, contains_data_from=None, contains_data_until=None)]
+        >>> dataset = all_optional_model.Dataset(short_name='person_data_v1', id='9662875c-c245-41de-b667-12ad2091a1ee', contains_data_from="2010-09-05", contains_data_until="2022-09-05")
+        >>> variables = [all_optional_model.Variable(short_name="pers", data_source=None, temporality_type=None, contains_data_from=None, contains_data_until=None)]
         >>> set_variables_inherit_from_dataset(dataset, variables)
 
         >>> variables[0].contains_data_from == dataset.contains_data_from
@@ -324,7 +339,9 @@ def num_obligatory_variables_fields_completed(variables: list) -> int:
     return num_completed
 
 
-def num_obligatory_variable_fields_completed(variable: model.Variable) -> int:
+def num_obligatory_variable_fields_completed(
+    variable: all_optional_model.Variable,
+) -> int:
     """Count the number of obligatory fields completed for one variable.
 
     This function calculates the total number of obligatory fields that have
@@ -428,85 +445,64 @@ def running_in_notebook() -> bool:
         return False
 
 
-def override_dataset_fields(
-    merged_metadata: all_optional_model.DatadocMetadata,
-    existing_metadata: all_optional_model.DatadocMetadata
-    | required_model.DatadocMetadata,
-) -> None:
-    """Overrides specific fields in the dataset of `merged_metadata` with values from the dataset of `existing_metadata`.
+def _ensure_encryption_parameters(
+    existing: list[dict[str, Any]] | None,
+    required: dict[str, Any],
+) -> list[dict[str, Any]]:
+    """Ensure required key/value pairs exist in parameters list."""
+    result = list(existing or [])
 
-    This function iterates over a predefined list of fields, `DATASET_FIELDS_FROM_EXISTING_METADATA`,
-    and sets the corresponding fields in the `merged_metadata.dataset` object to the values
-    from the `existing_metadata.dataset` object.
+    # Ensure each required key is present in at least one dict
+    for key, value in required.items():
+        if not any(key in d for d in result):
+            result.append({key: value})
 
-    Args:
-        merged_metadata: An instance of `DatadocMetadata` containing the dataset to be updated.
-        existing_metadata: An instance of `DatadocMetadata` containing the dataset whose values are used to update `merged_metadata.dataset`.
+    return result
 
-    Returns:
-        `None`.
-    """
-    if merged_metadata.dataset and existing_metadata.dataset:
-        # Override the fields as defined
-        for field in DATASET_FIELDS_FROM_EXISTING_METADATA:
-            setattr(
-                merged_metadata.dataset,
-                field,
-                getattr(existing_metadata.dataset, field),
-            )
-
-
-def merge_variables(
-    existing_metadata: OptionalDatadocMetadataType,
-    extracted_metadata: all_optional_model.DatadocMetadata,
-    merged_metadata: all_optional_model.DatadocMetadata,
-) -> all_optional_model.DatadocMetadata:
-    """Merges variables from the extracted metadata into the existing metadata and updates the merged metadata.
 
-    This function compares the variables from `extracted_metadata` with those in `existing_metadata`.
-    For each variable in `extracted_metadata`, it checks if a variable with the same `short_name` exists
-    in `existing_metadata`. If a match is found, it updates the existing variable with information from
-    `extracted_metadata`. If no match is found, the variable from `extracted_metadata` is directly added to `merged_metadata`.
-
-    Args:
-        existing_metadata: The metadata object containing the current state of variables.
-        extracted_metadata: The metadata object containing new or updated variables to merge.
-        merged_metadata: The metadata object that will contain the result of the merge.
+def set_default_values_pseudonymization(
+    variable: VariableType,
+    pseudonymization: PseudonymizationType | None,
+) -> None:
+    """Populate pseudonymization fields with defaults based on the encryption algorithm.
 
-    Returns:
-        all_optional_model.DatadocMetadata: The `merged_metadata` object containing variables from both `existing_metadata`
-        and `extracted_metadata`.
+    Updates the encryption key reference and encryption parameters if they are not set,
+    handling both PAPIS and DAED algorithms. Leaves unknown algorithms unchanged.
     """
-    if (
-        existing_metadata is not None
-        and existing_metadata.variables is not None
-        and extracted_metadata is not None
-        and extracted_metadata.variables is not None
-        and merged_metadata.variables is not None
-    ):
-        for extracted in extracted_metadata.variables:
-            existing = next(
-                (
-                    existing
-                    for existing in existing_metadata.variables
-                    if existing.short_name == extracted.short_name
-                ),
-                None,
-            )
-            if existing:
-                existing.id = (
-                    None  # Set to None so that it will be set assigned a fresh ID later
+    if pseudonymization is None:
+        return
+    if variable.pseudonymization is None:
+        variable.pseudonymization = pseudonymization
+    match pseudonymization.encryption_algorithm:
+        case EncryptionAlgorithm.PAPIS_ENCRYPTION_ALGORITHM.value:
+            if not pseudonymization.encryption_key_reference:
+                pseudonymization.encryption_key_reference = (
+                    PAPIS_ENCRYPTION_KEY_REFERENCE
                 )
-                existing.contains_data_from = (
-                    extracted.contains_data_from or existing.contains_data_from
+            base_params = {
+                ENCRYPTION_PARAMETER_KEY_ID: PAPIS_ENCRYPTION_KEY_REFERENCE,
+                ENCRYPTION_PARAMETER_STRATEGY: ENCRYPTION_PARAMETER_STRATEGY_SKIP,
+            }
+            if pseudonymization.stable_identifier_type == PAPIS_STABLE_IDENTIFIER_TYPE:
+                base_params[ENCRYPTION_PARAMETER_SNAPSHOT_DATE] = get_current_date()
+            pseudonymization.encryption_algorithm_parameters = (
+                _ensure_encryption_parameters(
+                    pseudonymization.encryption_algorithm_parameters,
+                    base_params,
                 )
-                existing.contains_data_until = (
-                    extracted.contains_data_until or existing.contains_data_until
+            )
+        case EncryptionAlgorithm.DAEAD_ENCRYPTION_ALGORITHM.value:
+            if not pseudonymization.encryption_key_reference:
+                pseudonymization.encryption_key_reference = (
+                    DAEAD_ENCRYPTION_KEY_REFERENCE
                 )
-                merged_metadata.variables.append(
-                    cast("datadoc_model.all_optional.model.Variable", existing)
+            pseudonymization.encryption_algorithm_parameters = (
+                _ensure_encryption_parameters(
+                    pseudonymization.encryption_algorithm_parameters,
+                    {
+                        ENCRYPTION_PARAMETER_KEY_ID: DAEAD_ENCRYPTION_KEY_REFERENCE,
+                    },
                 )
-            else:
-                # If there is no existing metadata for this variable, we just use what we have extracted
-                merged_metadata.variables.append(extracted)
-    return merged_metadata
+            )
+        case _:
+            pass

dapla_metadata/variable_definitions/variable_definition.py

@@ -49,6 +49,10 @@ from dapla_metadata.variable_definitions.exceptions import vardef_file_error_han
 
 logger = logging.getLogger(__name__)
 
+IDENTICAL_PATCH_ERROR_MESSAGE = (
+    "No changes detected in supported fields. Not creating identical patch."
+)
+
 
 class VariableDefinition(CompleteResponse):
     """A Variable Definition.
@@ -274,11 +278,14 @@ class VariableDefinition(CompleteResponse):
         Returns:
             VariableDefinition: Variable Definition with all details.
         """
+        new_patch = _read_file_to_model(
+            file_path or self.get_file_path(),
+            Patch,
+        )
+        if new_patch == Patch.from_dict(self.to_dict()):
+            raise ValueError(IDENTICAL_PATCH_ERROR_MESSAGE)
         return self.create_patch(
-            patch=_read_file_to_model(
-                file_path or self.get_file_path(),
-                Patch,
-            ),
+            patch=new_patch,
             valid_from=valid_from,
         )
 

{dapla_toolbelt_metadata-0.9.1.dist-info → dapla_toolbelt_metadata-0.9.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dapla-toolbelt-metadata
-Version: 0.9.1
+Version: 0.9.3
 Summary: Dapla Toolbelt Metadata
 Project-URL: homepage, https://github.com/statisticsnorway/dapla-toolbelt-metadata
 Project-URL: repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata

{dapla_toolbelt_metadata-0.9.1.dist-info → dapla_toolbelt_metadata-0.9.3.dist-info}/RECORD

@@ -5,9 +5,10 @@ dapla_metadata/_shared/enums.py,sha256=WHkH1d8xw41gOly6au_izZB1_-6XTcKu5rhBWUImj
 dapla_metadata/_shared/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dapla_metadata/dapla/__init__.py,sha256=tkapF-YwmruPPrKvN3pEoCZqb7xvJx_ogBM8XyGMuJI,130
 dapla_metadata/dapla/user_info.py,sha256=bENez-ICt9ySR8orYebO68Q3_2LkIW9QTL58DTctmEQ,4833
-dapla_metadata/datasets/__init__.py,sha256=TvzskpdFC6hGcC9_55URT5jr5wNAPzXuISd2UjJWM_8,280
+dapla_metadata/datasets/__init__.py,sha256=an-REJgi7N8-S1SCz-MYO_8as6fMe03WvhjRP_hWWkg,293
+dapla_metadata/datasets/_merge.py,sha256=Tk5wQz6xZGr8veUAHZb42O8HARU8ObBJ_E4afvVWdlo,12993
 dapla_metadata/datasets/code_list.py,sha256=JtCE-5Q8grAKvkn0KKjzeGhO-96O7yGsastbuoakreg,9057
-dapla_metadata/datasets/core.py,sha256=jwnV6kqS7GpS_9kVWbFz7J0TE-TiSSiqq_gSV4sE628,25774
+dapla_metadata/datasets/core.py,sha256=p-2OJsAEWCUqBlzn0YIYkK-pAgtvMROdoxXvCyjfWYs,20434
 dapla_metadata/datasets/dapla_dataset_path_info.py,sha256=WPeV_mwKk2B9sXd14SaP-kTb1bOQ_8W2KtrqOG7sJIY,26867
 dapla_metadata/datasets/dataset_parser.py,sha256=3dtRXNy1C8SfG8zTYWdY26nV4l-dG25IC_0J5t2bYwI,8285
 dapla_metadata/datasets/model_validation.py,sha256=pGT-jqaQQY4z7jz-7UQd0BQoTWDxDWPYAnDoRC2vd_c,6818
@@ -20,9 +21,9 @@ dapla_metadata/datasets/compatibility/model_backwards_compatibility.py,sha256=W5
 dapla_metadata/datasets/external_sources/__init__.py,sha256=qvIdXwqyEmXNUCB94ZtZXRzifdW4hiXASFFPtC70f6E,83
 dapla_metadata/datasets/external_sources/external_sources.py,sha256=9eIcOIUbaodNX1w9Tj2wl4U4wUmr5kF1R0i01fKUzGs,2974
 dapla_metadata/datasets/utility/__init__.py,sha256=pp6tUcgUbo8iq9OPtFKQrTbLuI3uY7NHptwWSTpasOU,33
-dapla_metadata/datasets/utility/constants.py,sha256=f9TfBN5aJbiKBQVpu8Whc0X-EMpXv43-Yu2L4KUDA4U,2353
-dapla_metadata/datasets/utility/enums.py,sha256=SpV4xlmP1YMaJPbmX03hqRLHUOhXIk5gquTeJ8G_5OE,432
-dapla_metadata/datasets/utility/utils.py,sha256=AjPWlg_8DJpqFQ8B2MaVJrB62EZ3dilvfRfc0DImjQI,18499
+dapla_metadata/datasets/utility/constants.py,sha256=94nGISL96rHvAndjHyaQEaJXNBnPAiRJN1slUaB03gM,2933
+dapla_metadata/datasets/utility/enums.py,sha256=i6dcxWya5k4LjLdGGIM_H37rRndizug3peaAgoE5UdM,652
+dapla_metadata/datasets/utility/utils.py,sha256=85Ms6jEcUuQUm-RRosscDVpvA5W4TOqiOZo2LAnXjFA,18301
 dapla_metadata/standards/__init__.py,sha256=n8jnMrudLuScSdfQ4UMJorc-Ptg3Y1-ilT8zAaQnM70,179
 dapla_metadata/standards/name_validator.py,sha256=6-DQE_EKVd6UjL--EXpFcZDQtusVbSFaWaUY-CfOV2c,9184
 dapla_metadata/standards/standard_validators.py,sha256=tcCiCI76wUVtMzXA2oCgdauZc0uGgUi11FKu-t7KGwQ,3767
@@ -33,7 +34,7 @@ dapla_metadata/variable_definitions/exceptions.py,sha256=ImB81bne-h45kX9lE5hIh80
 dapla_metadata/variable_definitions/vardef.py,sha256=WUpiKfvgFGPhMdjYSFSmdlXQKAolmRgW4-t-EocddQs,13934
 dapla_metadata/variable_definitions/vardok_id.py,sha256=8T23BUHyVQr5hovTVc2E4HVY7f7e_jdi3YL1qzMQgFw,1268
 dapla_metadata/variable_definitions/vardok_vardef_id_pair.py,sha256=8MDdd2-9L30MXkoQrk7NDcueaoxdeYie-TJhgoskTzk,1389
-dapla_metadata/variable_definitions/variable_definition.py,sha256=uSWvSuVDh5zmSXGUb7vitiNd0VThU5DzFV3Rd6gumYE,14620
+dapla_metadata/variable_definitions/variable_definition.py,sha256=tsTo_BObHgppSNf6lGbt_Vh88PypPF4pwA-vWDgFy9A,14869
 dapla_metadata/variable_definitions/_generated/.openapi-generator-ignore,sha256=x9lryVB5wtVEuKQ5GcZ94b10RgtkVXbtvWXOArO1XsM,169
 dapla_metadata/variable_definitions/_generated/README.md,sha256=Y4et1oAhZTCr7a-CZfLbIpyYnhKzpygNg-gj7qJ09Eg,7650
 dapla_metadata/variable_definitions/_generated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -89,7 +90,7 @@ dapla_metadata/variable_definitions/_utils/constants.py,sha256=zr5FNVCEz6TM9PVEr
 dapla_metadata/variable_definitions/_utils/files.py,sha256=JbPgPNQ7iA38juMqGEdcg5OjZZUwCb6NQtPL0AEspD0,10933
 dapla_metadata/variable_definitions/_utils/template_files.py,sha256=7fcc7yEHOl5JUZ698kqj4IiikXPHBi3SrAVOk4wqQtw,3308
 dapla_metadata/variable_definitions/_utils/variable_definition_files.py,sha256=sGhcSpckR9NtYGNh2oVkiCd5SI3bbJEBhc1PA2uShs0,4701
-dapla_toolbelt_metadata-0.9.1.dist-info/METADATA,sha256=aExcayYhHGSrYtImpcv7e5vZFFvAaSuAtbpheyCOeN8,4723
-dapla_toolbelt_metadata-0.9.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dapla_toolbelt_metadata-0.9.1.dist-info/licenses/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
-dapla_toolbelt_metadata-0.9.1.dist-info/RECORD,,
+dapla_toolbelt_metadata-0.9.3.dist-info/METADATA,sha256=aCJzU5NSK7_yY9lu5R9KnO-cE6P6o_-gc_32CS6qeKU,4723
+dapla_toolbelt_metadata-0.9.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dapla_toolbelt_metadata-0.9.3.dist-info/licenses/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
+dapla_toolbelt_metadata-0.9.3.dist-info/RECORD,,