dapla-toolbelt-metadata 0.9.2__tar.gz → 0.9.3__tar.gz

{dapla_toolbelt_metadata-0.9.2 → dapla_toolbelt_metadata-0.9.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dapla-toolbelt-metadata
-Version: 0.9.2
+Version: 0.9.3
 Summary: Dapla Toolbelt Metadata
 Project-URL: homepage, https://github.com/statisticsnorway/dapla-toolbelt-metadata
 Project-URL: repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata

{dapla_toolbelt_metadata-0.9.2 → dapla_toolbelt_metadata-0.9.3}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "dapla-toolbelt-metadata"
 description = "Dapla Toolbelt Metadata"
 license = "MIT"
-version = "0.9.2"
+version = "0.9.3"
 classifiers = ["Development Status :: 4 - Beta"]
 readme = "README.md"
 authors = [{ name = "Statistics Norway", email = "metadata@ssb.no" }]

{dapla_toolbelt_metadata-0.9.2 → dapla_toolbelt_metadata-0.9.3}/src/dapla_metadata/datasets/core.py

@@ -46,6 +46,7 @@ from dapla_metadata.datasets.utility.utils import (
 )
 from dapla_metadata.datasets.utility.utils import set_dataset_owner
 from dapla_metadata.datasets.utility.utils import set_default_values_dataset
+from dapla_metadata.datasets.utility.utils import set_default_values_pseudonymization
 from dapla_metadata.datasets.utility.utils import set_default_values_variables
 
 if TYPE_CHECKING:
@@ -444,9 +445,12 @@ class Datadoc:
         variable_short_name: str,
         pseudonymization: all_optional_model.Pseudonymization | None = None,
     ) -> None:
-        """Adds a new pseudo variable to the list of pseudonymized variables also sets is_personal_data to true.
+        """Adds a new pseudo variable to the list of pseudonymized variables.
 
-        If there is no pseudonymization supplied an empty Pseudonymization structure will be added to the model.
+        If `pseudonymization` is not supplied, an empty Pseudonymization structure
+        will be created and assigned to the variable.
+        If an encryption algorithm is recognized (one of the standard Dapla algorithms), default values are filled
+        for any missing fields.
 
         Args:
             variable_short_name: The short name for the variable that one wants to update the pseudo for.
@@ -454,15 +458,15 @@ class Datadoc:
 
         """
         variable = self.variables_lookup[variable_short_name]
-        variable.pseudonymization = (
-            pseudonymization or all_optional_model.Pseudonymization()
-        )
+        if pseudonymization:
+            set_default_values_pseudonymization(variable, pseudonymization)
+        else:
+            variable.pseudonymization = all_optional_model.Pseudonymization()
 
     def remove_pseudonymization(self, variable_short_name: str) -> None:
         """Removes a pseudo variable by using the shortname.
 
         Updates the pseudo variable lookup by creating a new one.
-        Sets is_personal_data to non pseudonymized encrypted personal data.
 
         Args:
             variable_short_name: The short name for the variable that one wants to remove the pseudo for.

{dapla_toolbelt_metadata-0.9.2 → dapla_toolbelt_metadata-0.9.3}/src/dapla_metadata/datasets/utility/constants.py

@@ -47,6 +47,12 @@ OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS = [
     "temporality_type",
 ]
 
+OBLIGATORY_VARIABLES_PESUODONYMIZATION_IDENTIFIERS = [
+    "encryption_algorithm",
+    "encryption_key_refrence",
+]
+
+
 OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS_MULTILANGUAGE = [
     "name",
 ]
@@ -91,3 +97,11 @@ METADATA_DOCUMENT_FILE_SUFFIX = "__DOC.json"
 DATADOC_STATISTICAL_SUBJECT_SOURCE_URL = (
     "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
 )
+
+PAPIS_STABLE_IDENTIFIER_TYPE = "FREG_SNR"
+PAPIS_ENCRYPTION_KEY_REFERENCE = "papis-common-key-1"
+DAEAD_ENCRYPTION_KEY_REFERENCE = "ssb-common-key-1"
+ENCRYPTION_PARAMETER_SNAPSHOT_DATE = "snapshotDate"
+ENCRYPTION_PARAMETER_KEY_ID = "keyId"
+ENCRYPTION_PARAMETER_STRATEGY = "strategy"
+ENCRYPTION_PARAMETER_STRATEGY_SKIP = "skip"

{dapla_toolbelt_metadata-0.9.2 → dapla_toolbelt_metadata-0.9.3}/src/dapla_metadata/datasets/utility/enums.py

@@ -14,3 +14,10 @@ class SupportedLanguages(str, Enum):
     NORSK_BOKMÅL = "nb"  # noqa: PLC2401 the listed problems do not apply in this case
     NORSK_NYNORSK = "nn"
     ENGLISH = "en"
+
+
+class EncryptionAlgorithm(str, Enum):
+    """Encryption algorithm values for pseudonymization algoprithms offered on Dapla."""
+
+    PAPIS_ENCRYPTION_ALGORITHM = "TINK-FPE"
+    DAEAD_ENCRYPTION_ALGORITHM = "TINK-DAEAD"

{dapla_toolbelt_metadata-0.9.2 → dapla_toolbelt_metadata-0.9.3}/src/dapla_metadata/datasets/utility/utils.py

@@ -4,6 +4,7 @@ import datetime  # import is needed in xdoctest
 import logging
 import pathlib
 import uuid
+from typing import Any
 from typing import TypeAlias
 
 import datadoc_model.all_optional.model as all_optional_model
@@ -12,12 +13,16 @@ import google.auth
 from cloudpathlib import CloudPath
 from cloudpathlib import GSClient
 from cloudpathlib import GSPath
-from datadoc_model import model
 from datadoc_model.all_optional.model import Assessment
 from datadoc_model.all_optional.model import DataSetState
 from datadoc_model.all_optional.model import VariableRole
 
 from dapla_metadata.dapla import user_info
+from dapla_metadata.datasets.utility.constants import DAEAD_ENCRYPTION_KEY_REFERENCE
+from dapla_metadata.datasets.utility.constants import ENCRYPTION_PARAMETER_KEY_ID
+from dapla_metadata.datasets.utility.constants import ENCRYPTION_PARAMETER_SNAPSHOT_DATE
+from dapla_metadata.datasets.utility.constants import ENCRYPTION_PARAMETER_STRATEGY
+from dapla_metadata.datasets.utility.constants import ENCRYPTION_PARAMETER_STRATEGY_SKIP
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
 from dapla_metadata.datasets.utility.constants import (
     OBLIGATORY_DATASET_METADATA_IDENTIFIERS,
@@ -31,6 +36,9 @@ from dapla_metadata.datasets.utility.constants import (
 from dapla_metadata.datasets.utility.constants import (
     OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS_MULTILANGUAGE,
 )
+from dapla_metadata.datasets.utility.constants import PAPIS_ENCRYPTION_KEY_REFERENCE
+from dapla_metadata.datasets.utility.constants import PAPIS_STABLE_IDENTIFIER_TYPE
+from dapla_metadata.datasets.utility.enums import EncryptionAlgorithm
 
 logger = logging.getLogger(__name__)
 
@@ -39,12 +47,20 @@ DatadocMetadataType: TypeAlias = (
 )
 DatasetType: TypeAlias = all_optional_model.Dataset | required_model.Dataset
 VariableType: TypeAlias = all_optional_model.Variable | required_model.Variable
+PseudonymizationType: TypeAlias = (
+    all_optional_model.Pseudonymization | required_model.Pseudonymization
+)
 VariableListType: TypeAlias = (
     list[all_optional_model.Variable] | list[required_model.Variable]
 )
 OptionalDatadocMetadataType: TypeAlias = DatadocMetadataType | None
 
 
+def get_current_date() -> str:
+    """Return a current date as str."""
+    return datetime.datetime.now(tz=datetime.timezone.utc).date().isoformat()
+
+
 def get_timestamp_now() -> datetime.datetime:
     """Return a timestamp for the current moment."""
     return datetime.datetime.now(tz=datetime.timezone.utc)
@@ -109,7 +125,7 @@ def set_default_values_variables(variables: list) -> None:
         variables: A list of variable objects to set default values on.
 
     Example:
-        >>> variables = [model.Variable(short_name="pers",id=None, is_personal_data = None), model.Variable(short_name="fnr",id='9662875c-c245-41de-b667-12ad2091a1ee', is_personal_data=True)]
+        >>> variables = [all_optional_model.Variable(short_name="pers",id=None, is_personal_data = None), all_optional_model.Variable(short_name="fnr",id='9662875c-c245-41de-b667-12ad2091a1ee', is_personal_data=True)]
         >>> set_default_values_variables(variables)
         >>> isinstance(variables[0].id, uuid.UUID)
         True
@@ -138,7 +154,7 @@ def set_default_values_dataset(
         dataset: The dataset object to set default values on.
 
     Example:
-        >>> dataset = model.Dataset(id=None)
+        >>> dataset = all_optional_model.Dataset(id=None)
         >>> set_default_values_dataset(dataset)
         >>> dataset.id is not None
         True
@@ -176,8 +192,8 @@ def set_variables_inherit_from_dataset(
         variables: A list of variable objects to update with dataset values.
 
     Example:
-        >>> dataset = model.Dataset(short_name='person_data_v1', id='9662875c-c245-41de-b667-12ad2091a1ee', contains_data_from="2010-09-05", contains_data_until="2022-09-05")
-        >>> variables = [model.Variable(short_name="pers", data_source=None, temporality_type=None, contains_data_from=None, contains_data_until=None)]
+        >>> dataset = all_optional_model.Dataset(short_name='person_data_v1', id='9662875c-c245-41de-b667-12ad2091a1ee', contains_data_from="2010-09-05", contains_data_until="2022-09-05")
+        >>> variables = [all_optional_model.Variable(short_name="pers", data_source=None, temporality_type=None, contains_data_from=None, contains_data_until=None)]
         >>> set_variables_inherit_from_dataset(dataset, variables)
 
         >>> variables[0].contains_data_from == dataset.contains_data_from
@@ -323,7 +339,9 @@ def num_obligatory_variables_fields_completed(variables: list) -> int:
     return num_completed
 
 
-def num_obligatory_variable_fields_completed(variable: model.Variable) -> int:
+def num_obligatory_variable_fields_completed(
+    variable: all_optional_model.Variable,
+) -> int:
     """Count the number of obligatory fields completed for one variable.
 
     This function calculates the total number of obligatory fields that have
@@ -425,3 +443,66 @@ def running_in_notebook() -> bool:
         # interpreters and will throw a NameError. Therefore we're not running
         # in Jupyter.
         return False
+
+
+def _ensure_encryption_parameters(
+    existing: list[dict[str, Any]] | None,
+    required: dict[str, Any],
+) -> list[dict[str, Any]]:
+    """Ensure required key/value pairs exist in parameters list."""
+    result = list(existing or [])
+
+    # Ensure each required key is present in at least one dict
+    for key, value in required.items():
+        if not any(key in d for d in result):
+            result.append({key: value})
+
+    return result
+
+
+def set_default_values_pseudonymization(
+    variable: VariableType,
+    pseudonymization: PseudonymizationType | None,
+) -> None:
+    """Populate pseudonymization fields with defaults based on the encryption algorithm.
+
+    Updates the encryption key reference and encryption parameters if they are not set,
+    handling both PAPIS and DAED algorithms. Leaves unknown algorithms unchanged.
+    """
+    if pseudonymization is None:
+        return
+    if variable.pseudonymization is None:
+        variable.pseudonymization = pseudonymization
+    match pseudonymization.encryption_algorithm:
+        case EncryptionAlgorithm.PAPIS_ENCRYPTION_ALGORITHM.value:
+            if not pseudonymization.encryption_key_reference:
+                pseudonymization.encryption_key_reference = (
+                    PAPIS_ENCRYPTION_KEY_REFERENCE
+                )
+            base_params = {
+                ENCRYPTION_PARAMETER_KEY_ID: PAPIS_ENCRYPTION_KEY_REFERENCE,
+                ENCRYPTION_PARAMETER_STRATEGY: ENCRYPTION_PARAMETER_STRATEGY_SKIP,
+            }
+            if pseudonymization.stable_identifier_type == PAPIS_STABLE_IDENTIFIER_TYPE:
+                base_params[ENCRYPTION_PARAMETER_SNAPSHOT_DATE] = get_current_date()
+            pseudonymization.encryption_algorithm_parameters = (
+                _ensure_encryption_parameters(
+                    pseudonymization.encryption_algorithm_parameters,
+                    base_params,
+                )
+            )
+        case EncryptionAlgorithm.DAEAD_ENCRYPTION_ALGORITHM.value:
+            if not pseudonymization.encryption_key_reference:
+                pseudonymization.encryption_key_reference = (
+                    DAEAD_ENCRYPTION_KEY_REFERENCE
+                )
+            pseudonymization.encryption_algorithm_parameters = (
+                _ensure_encryption_parameters(
+                    pseudonymization.encryption_algorithm_parameters,
+                    {
+                        ENCRYPTION_PARAMETER_KEY_ID: DAEAD_ENCRYPTION_KEY_REFERENCE,
+                    },
+                )
+            )
+        case _:
+            pass

{dapla_toolbelt_metadata-0.9.2 → dapla_toolbelt_metadata-0.9.3}/src/dapla_metadata/variable_definitions/variable_definition.py

@@ -49,6 +49,10 @@ from dapla_metadata.variable_definitions.exceptions import vardef_file_error_han
 
 logger = logging.getLogger(__name__)
 
+IDENTICAL_PATCH_ERROR_MESSAGE = (
+    "No changes detected in supported fields. Not creating identical patch."
+)
+
 
 class VariableDefinition(CompleteResponse):
     """A Variable Definition.
@@ -274,11 +278,14 @@ class VariableDefinition(CompleteResponse):
         Returns:
             VariableDefinition: Variable Definition with all details.
         """
+        new_patch = _read_file_to_model(
+            file_path or self.get_file_path(),
+            Patch,
+        )
+        if new_patch == Patch.from_dict(self.to_dict()):
+            raise ValueError(IDENTICAL_PATCH_ERROR_MESSAGE)
         return self.create_patch(
-            patch=_read_file_to_model(
-                file_path or self.get_file_path(),
-                Patch,
-            ),
+            patch=new_patch,
             valid_from=valid_from,
         )
 

{dapla_toolbelt_metadata-0.9.2 → dapla_toolbelt_metadata-0.9.3}/tests/datasets/test_datadoc_metadata.py

@@ -19,7 +19,6 @@ from datadoc_model.all_optional.model import Dataset
 from datadoc_model.all_optional.model import DataSetState
 from datadoc_model.all_optional.model import DataSetStatus
 from datadoc_model.all_optional.model import DataType
-from datadoc_model.all_optional.model import Pseudonymization
 from datadoc_model.all_optional.model import Variable
 from datadoc_model.all_optional.model import VariableRole
 from pydantic import ValidationError
@@ -39,7 +38,6 @@ from tests.datasets.constants import TEST_EXISTING_METADATA_WITH_VALID_ID_DIRECT
 from tests.datasets.constants import TEST_NAMING_STANDARD_COMPATIBLE_DATASET
 from tests.datasets.constants import TEST_PARQUET_FILEPATH
 from tests.datasets.constants import TEST_PROCESSED_DATA_POPULATION_DIRECTORY
-from tests.datasets.constants import TEST_PSEUDO_DIRECTORY
 from tests.datasets.constants import TEST_RESOURCES_DIRECTORY
 
 if TYPE_CHECKING:
@@ -583,93 +581,3 @@ def test_merge_with_fewer_variables_in_existing_metadata(tmp_path):
         "bankinnskudd",
         "dato",
     ]
-
-
-@pytest.mark.parametrize(
-    "existing_metadata_path",
-    [TEST_PSEUDO_DIRECTORY / "dataset_and_pseudo"],
-)
-def test_add_pseudo_variable(
-    existing_metadata_file: Path,  # noqa: ARG001
-    metadata: Datadoc,
-):
-    test_variable = "sykepenger"
-    is_personal_data = metadata.variables_lookup[test_variable].is_personal_data
-    metadata.add_pseudonymization(test_variable)
-    assert metadata.variables_lookup[test_variable].pseudonymization is not None
-    assert metadata.variables_lookup[test_variable].is_personal_data == is_personal_data
-
-
-@pytest.mark.parametrize(
-    "existing_metadata_path",
-    [TEST_PSEUDO_DIRECTORY / "dataset_and_pseudo"],
-)
-def test_add_pseudo_variable_non_existent_variable_name(
-    existing_metadata_file: Path,  # noqa: ARG001
-    metadata: Datadoc,
-):
-    with pytest.raises(KeyError):
-        metadata.add_pseudonymization("new_pseudo_variable")
-
-
-@pytest.mark.parametrize(
-    "existing_metadata_path",
-    [TEST_PSEUDO_DIRECTORY / "dataset_and_pseudo"],
-)
-def test_existing_metadata_file_update_pseudonymization(
-    existing_metadata_file: Path,  # noqa: ARG001
-    metadata: Datadoc,
-):
-    metadata.add_pseudonymization("pers_id")
-    variable = metadata.variables_lookup["pers_id"]
-
-    assert variable.pseudonymization is not None
-    assert variable.pseudonymization.encryption_algorithm is None
-    variable.pseudonymization.encryption_algorithm = "new_encryption_algorithm"
-    assert variable.pseudonymization.encryption_algorithm == "new_encryption_algorithm"
-
-
-@pytest.mark.parametrize(
-    "existing_metadata_path",
-    [TEST_PSEUDO_DIRECTORY / "dataset_and_pseudo"],
-)
-def test_update_pseudo(
-    existing_metadata_file: Path,  # noqa: ARG001
-    metadata: Datadoc,
-):
-    variable = metadata.variables_lookup["pers_id"]
-
-    pseudo = Pseudonymization(encryption_algorithm="new_encryption_algorithm")
-
-    metadata.add_pseudonymization("pers_id", pseudo)
-
-    assert variable.pseudonymization is not None
-    assert variable.pseudonymization.encryption_algorithm == "new_encryption_algorithm"
-    assert variable.pseudonymization.encryption_algorithm_parameters is None
-
-
-@pytest.mark.parametrize(
-    "existing_metadata_path",
-    [TEST_PSEUDO_DIRECTORY / "dataset_and_pseudo"],
-)
-def test_remove_pseudo_variable(
-    existing_metadata_file: Path,  # noqa: ARG001
-    metadata: Datadoc,
-):
-    test_variable = "alm_inntekt"
-    is_personal_data = metadata.variables_lookup[test_variable].is_personal_data
-    metadata.remove_pseudonymization(test_variable)
-    assert metadata.variables_lookup[test_variable].is_personal_data == is_personal_data
-    assert metadata.variables_lookup[test_variable].pseudonymization is None
-
-
-@pytest.mark.parametrize(
-    "existing_metadata_path",
-    [TEST_PSEUDO_DIRECTORY / "dataset_and_pseudo"],
-)
-def test_remove_pseudo_variable_non_existent_variable_name(
-    existing_metadata_file: Path,  # noqa: ARG001
-    metadata: Datadoc,
-):
-    with pytest.raises(KeyError):
-        metadata.remove_pseudonymization("fnr")