dapla-toolbelt-metadata 0.7.2__tar.gz → 0.8.1__tar.gz

{dapla_toolbelt_metadata-0.7.2 → dapla_toolbelt_metadata-0.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dapla-toolbelt-metadata
-Version: 0.7.2
+Version: 0.8.1
 Summary: Dapla Toolbelt Metadata
 License: MIT
 Author: Team Metadata
@@ -24,7 +24,7 @@ Requires-Dist: pyjwt (>=2.8.0)
 Requires-Dist: python-dotenv (>=1.0.1)
 Requires-Dist: requests (>=2.31.0)
 Requires-Dist: ruamel-yaml (>=0.18.10)
-Requires-Dist: ssb-datadoc-model (==6.1.0)
+Requires-Dist: ssb-datadoc-model (==7.0.1)
 Requires-Dist: ssb-klass-python (>=1.0.1)
 Requires-Dist: typing-extensions (>=4.12.2)
 Project-URL: Changelog, https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases

{dapla_toolbelt_metadata-0.7.2 → dapla_toolbelt_metadata-0.8.1}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "dapla-toolbelt-metadata"
 description = "Dapla Toolbelt Metadata"
 license = "MIT"
-version = "0.7.2"
+version = "0.8.1"
 dynamic = ["classifiers"]
 readme = "README.md"
 authors = [{ name = "Team Metadata", email = "metadata@ssb.no" }]
@@ -17,7 +17,7 @@ dependencies = [
     'cloudpathlib[gs] >=0.17.0',
     'pyjwt >=2.8.0',
     'ssb-klass-python >=1.0.1',
-    'ssb-datadoc-model ==6.1.0',
+    'ssb-datadoc-model ==7.0.1',
     'typing-extensions >=4.12.2',
     'ruamel-yaml >=0.18.10',
     'google-auth >=2.38.0',

{dapla_toolbelt_metadata-0.7.2 → dapla_toolbelt_metadata-0.8.1}/src/dapla_metadata/datasets/core.py

@@ -32,7 +32,6 @@ from dapla_metadata.datasets.utility.constants import INCONSISTENCIES_MESSAGE
 from dapla_metadata.datasets.utility.constants import METADATA_DOCUMENT_FILE_SUFFIX
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
-from dapla_metadata.datasets.utility.utils import ExistingPseudonymizationMetadataType
 from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
 from dapla_metadata.datasets.utility.utils import calculate_percentage
 from dapla_metadata.datasets.utility.utils import derive_assessment_from_state
@@ -119,9 +118,7 @@ class Datadoc:
         self.dataset_path: pathlib.Path | CloudPath | None = None
         self.dataset = all_optional_model.Dataset()
         self.variables: list = []
-        self.pseudo_variables: list[all_optional_model.PseudoVariable] = []
         self.variables_lookup: dict[str, all_optional_model.Variable] = {}
-        self.pseudo_variables_lookup: dict[str, all_optional_model.PseudoVariable] = {}
         self.explicitly_defined_metadata_document = False
         self.dataset_consistency_status: list = []
         if metadata_document_path:
@@ -161,19 +158,12 @@ class Datadoc:
         """
         extracted_metadata: all_optional_model.DatadocMetadata | None = None
         existing_metadata: OptionalDatadocMetadataType = None
-        existing_pseudonymization: ExistingPseudonymizationMetadataType = None
 
         if self.metadata_document and self.metadata_document.exists():
             existing_metadata = self._extract_metadata_from_existing_document(
                 self.metadata_document,
             )
 
-            existing_pseudonymization = (
-                self._extract_pseudonymization_from_existing_document(
-                    self.metadata_document,
-                )
-            )
-
         if (
             self.dataset_path is not None
             and self.dataset == all_optional_model.Dataset()
@@ -221,14 +211,10 @@ class Datadoc:
         else:
             self._set_metadata(existing_metadata or extracted_metadata)
 
-        if existing_pseudonymization:
-            self._set_pseudonymization_metadata(existing_pseudonymization)
-
         set_default_values_variables(self.variables)
         set_default_values_dataset(cast("all_optional_model.Dataset", self.dataset))
         set_dataset_owner(self.dataset)
         self._create_variables_lookup()
-        self._create_pseudo_variables_lookup()
 
     def _get_existing_file_path(
         self,
@@ -255,32 +241,11 @@ class Datadoc:
         self.dataset = cast("all_optional_model.Dataset", merged_metadata.dataset)
         self.variables = merged_metadata.variables
 
-    def _set_pseudonymization_metadata(
-        self,
-        existing_pseudonymization: ExistingPseudonymizationMetadataType,
-    ) -> None:
-        if not existing_pseudonymization or not (
-            existing_pseudonymization.pseudo_variables is not None
-        ):
-            msg = "Error reading pseudonymization metadata"
-            logger.error(msg)
-            return
-        self.pseudo_variables = cast(
-            "list[all_optional_model.PseudoVariable]",
-            existing_pseudonymization.pseudo_variables,
-        )
-
     def _create_variables_lookup(self) -> None:
         self.variables_lookup = {
             v.short_name: v for v in self.variables if v.short_name
         }
 
-    def _create_pseudo_variables_lookup(self) -> None:
-        if self.pseudo_variables:
-            self.pseudo_variables_lookup = {
-                v.short_name: v for v in self.pseudo_variables if v.short_name
-            }
-
     @staticmethod
     def _check_dataset_consistency(
         new_dataset_path: Path | CloudPath,
@@ -463,53 +428,6 @@ class Datadoc:
             )
             return None
 
-    def _extract_pseudonymization_from_existing_document(
-        self,
-        document: pathlib.Path | CloudPath,
-    ) -> (
-        all_optional_model.PseudonymizationMetadata
-        | required_model.PseudonymizationMetadata
-        | None
-    ):
-        """Read pseudo metadata from an existing metadata document.
-
-        If there is pseudo metadata in the document supplied, the method validates and returns the pseudonymization structure.
-
-        Args:
-            document: A path to the existing metadata document.
-
-        Raises:
-            json.JSONDecodeError: If the metadata document cannot be parsed.
-            pydantic.ValidationError: If the data does not successfully validate.
-        """
-        metadata_model = (
-            required_model
-            if self.validate_required_fields_on_existing_metadata
-            else all_optional_model
-        )
-
-        try:
-            with document.open(mode="r", encoding="utf-8") as file:
-                fresh_metadata = json.load(file)
-        except json.JSONDecodeError:
-            logger.warning(
-                "Could not open existing metadata file %s.",
-                document,
-                exc_info=True,
-            )
-            return None
-
-        if not is_metadata_in_container_structure(fresh_metadata):
-            return None
-
-        pseudonymization_metadata = fresh_metadata.get("pseudonymization")
-        if pseudonymization_metadata is None:
-            return None
-
-        return metadata_model.PseudonymizationMetadata.model_validate_json(
-            json.dumps(pseudonymization_metadata),
-        )
-
     def _extract_subject_field_from_path(
         self,
         dapla_dataset_path_info: DaplaDatasetPathInfo,
@@ -627,13 +545,6 @@ class Datadoc:
         )
         if self.container:
             self.container.datadoc = datadoc
-            if not self.container.pseudonymization:
-                self.container.pseudonymization = (
-                    all_optional_model.PseudonymizationMetadata(
-                        pseudo_dataset=all_optional_model.PseudoDataset()
-                    )
-                )
-            self.container.pseudonymization.pseudo_variables = self.pseudo_variables
         else:
             self.container = all_optional_model.MetadataContainer(datadoc=datadoc)
         if self.metadata_document:
@@ -664,45 +575,35 @@ class Datadoc:
         ) + num_obligatory_variables_fields_completed(self.variables)
         return calculate_percentage(num_set_fields, num_all_fields)
 
-    def add_pseudo_variable(self, variable_short_name: str) -> None:
-        """Adds a new pseudo variable to the list of pseudonymized variables.
+    def add_pseudonymization(
+        self,
+        variable_short_name: str,
+        pseudonymization: all_optional_model.Pseudonymization | None = None,
+    ) -> None:
+        """Adds a new pseudo variable to the list of pseudonymized variables also sets is_personal_data to true.
+
+        If there is no pseudonymization supplied an empty Pseudonymization structure will be added to the model.
+
+        Args:
+            variable_short_name: The short name for the variable that one wants to update the pseudo for.
+            pseudonymization: The updated pseudonymization.
 
-        Sets is_personal_data to pseudonymized encrypted personal data.
         """
-        if self.variables_lookup[variable_short_name] is not None:
-            pseudo_variable = all_optional_model.PseudoVariable(
-                short_name=variable_short_name
-            )
-            self.pseudo_variables.append(pseudo_variable)
-            self.pseudo_variables_lookup[variable_short_name] = pseudo_variable
-            self.variables_lookup[
-                variable_short_name
-            ].is_personal_data = (
-                all_optional_model.IsPersonalData.PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA
-            )
+        variable = self.variables_lookup[variable_short_name]
+        variable.pseudonymization = (
+            pseudonymization or all_optional_model.Pseudonymization()
+        )
+        variable.is_personal_data = True
 
-    def remove_pseudo_variable(self, variable_short_name: str) -> None:
+    def remove_pseudonymization(self, variable_short_name: str) -> None:
         """Removes a pseudo variable by using the shortname.
 
         Updates the pseudo variable lookup by creating a new one.
         Sets is_personal_data to non pseudonymized encrypted personal data.
+
+        Args:
+            variable_short_name: The short name for the variable that one wants to remove the pseudo for.
         """
-        if self.pseudo_variables_lookup[variable_short_name] is not None:
-            pseudo_variable = self.get_pseudo_variable(variable_short_name)
-
-            if pseudo_variable is not None:
-                self.pseudo_variables = [
-                    pseudo_variable
-                    for pseudo_variable in self.pseudo_variables
-                    if pseudo_variable.short_name != variable_short_name
-                ]
-            self._create_pseudo_variables_lookup()
-            self.variables_lookup[
-                variable_short_name
-            ].is_personal_data = all_optional_model.IsPersonalData.NON_PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA
-
-    def get_pseudo_variable(
-        self, variable_short_name: str
-    ) -> all_optional_model.PseudoVariable | None:
-        """Finds a pseudo variable by shortname."""
-        return self.pseudo_variables_lookup.get(variable_short_name)
+        if self.variables_lookup[variable_short_name].pseudonymization is not None:
+            self.variables_lookup[variable_short_name].pseudonymization = None
+            self.variables_lookup[variable_short_name].is_personal_data = False

{dapla_toolbelt_metadata-0.7.2 → dapla_toolbelt_metadata-0.8.1}/src/dapla_metadata/datasets/model_backwards_compatibility.py

@@ -13,6 +13,7 @@ A test must also be implemented for each new version.
 
 from __future__ import annotations
 
+import logging
 from collections import OrderedDict
 from dataclasses import dataclass
 from datetime import datetime
@@ -22,10 +23,13 @@ from typing import Any
 
 import arrow
 
+logger = logging.getLogger(__name__)
+
 if TYPE_CHECKING:
     from collections.abc import Callable
 
 VERSION_FIELD_NAME = "document_version"
+PSEUDONYMIZATION_KEY = "pseudonymization"
 
 
 class UnknownModelVersionError(Exception):
@@ -187,6 +191,106 @@ def _cast_to_date_type(value_to_update: str | None) -> str | None:
     )
 
 
+def convert_is_personal_data(supplied_metadata: dict[str, Any]) -> None:
+    """Convert 'is_personal_data' values in the supplied metadata to boolean.
+
+    Iterates over variables in the supplied metadata and updates the
+    'is_personal_data' field:
+      - Sets it to True for NON_PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA and PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA.
+      - Sets it to False for NOT_PERSONAL_DATA.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+    """
+    for variable in supplied_metadata["datadoc"]["variables"]:
+        value = variable["is_personal_data"]
+        if value in (
+            "NON_PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA",
+            "PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA",
+        ):
+            variable["is_personal_data"] = True
+        elif value == "NOT_PERSONAL_DATA":
+            variable["is_personal_data"] = False
+
+
+def copy_pseudonymization_metadata(supplied_metadata: dict[str, Any]) -> None:
+    """Copies pseudonymization metadata from the old pseudonymization section into the corresponding variable.
+
+    For each variable in `supplied_metadata["datadoc"]["variables"]` that has a matching
+    `short_name` in `supplied_metadata["pseudonymization"]["pseudo_variables"]`, this
+    function copies the following fields into the variable's 'pseudonymization' dictionary:
+
+        - stable_identifier_type
+        - stable_identifier_version
+        - encryption_algorithm
+        - encryption_key_reference
+        - encryption_algorithm_parameters
+
+    From the pseudo_dataset the value dataset_pseudo_time is copied to each variable as pseudonymization_time.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+    """
+    pseudo_vars = supplied_metadata.get(PSEUDONYMIZATION_KEY, {}).get(
+        "pseudo_variables", []
+    )
+    pseudo_dataset = (
+        supplied_metadata.get(PSEUDONYMIZATION_KEY, {}).get("pseudo_dataset") or {}
+    )
+    pseudo_time = pseudo_dataset.get("dataset_pseudo_time", None)
+    datadoc_vars = supplied_metadata.get("datadoc", {}).get("variables", [])
+    pseudo_lookup = {var.get("short_name"): var for var in pseudo_vars}
+
+    for variable in datadoc_vars:
+        short_name = variable.get("short_name")
+        if short_name in pseudo_lookup:
+            pseudo_var = pseudo_lookup[short_name]
+            variable[PSEUDONYMIZATION_KEY] = variable.get(
+                PSEUDONYMIZATION_KEY, {}
+            ).copy()
+
+            for field in [
+                "stable_identifier_type",
+                "stable_identifier_version",
+                "encryption_algorithm",
+                "encryption_key_reference",
+                "encryption_algorithm_parameters",
+            ]:
+                variable[PSEUDONYMIZATION_KEY][field] = pseudo_var[field]
+            variable[PSEUDONYMIZATION_KEY]["pseudonymization_time"] = pseudo_time
+
+        else:
+            variable[PSEUDONYMIZATION_KEY] = None
+
+
+def handle_version_4_0_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 5.0.1.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 5.0.1. Specifically, it:
+    - Copies pseudonymization metadata if pseudonymization is enabled.
+    - Converts the 'is_personal_data' fields to be a bool.
+    - Updates the 'document_version' field in the 'datadoc' section to "5.0.1".
+    - All 'pseudonymization' from the container is removed.
+    - It also updates the container version to 1.0.0 from 0.0.1
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+    """
+    if supplied_metadata.get(PSEUDONYMIZATION_KEY):
+        copy_pseudonymization_metadata(supplied_metadata)
+
+    convert_is_personal_data(supplied_metadata)
+
+    supplied_metadata["datadoc"]["document_version"] = "5.0.1"
+    _remove_element_from_model(supplied_metadata, PSEUDONYMIZATION_KEY)
+    supplied_metadata["document_version"] = "1.0.0"
+    return supplied_metadata
+
+
 def handle_version_3_3_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
     """Handle breaking changes for version 3.3.0.
 
@@ -195,17 +299,19 @@ def handle_version_3_3_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
     'direct_person_identifying' field from each variable in 'datadoc.variables'
     and updates the 'document_version' field to "4.0.0".
 
+    Version 4.0.0 used an enum for is_personal_data, however this was changed to a bool again for version 5.0.1.
+    We skip setting the enum here and just keep the value it has.
+
     Args:
         supplied_metadata: The metadata dictionary to be updated.
 
     Returns:
         The updated metadata dictionary.
     """
-    for i in range(len(supplied_metadata["datadoc"]["variables"])):
-        _remove_element_from_model(
-            supplied_metadata["datadoc"]["variables"][i],
-            "direct_person_identifying",
-        )
+    for variable in supplied_metadata["datadoc"]["variables"]:
+        variable["is_personal_data"] = variable["direct_person_identifying"]
+        _remove_element_from_model(variable, "direct_person_identifying")
+
     supplied_metadata["datadoc"]["document_version"] = "4.0.0"
     return supplied_metadata
 
@@ -457,7 +563,8 @@ BackwardsCompatibleVersion(version="2.2.0", handler=handle_version_2_2_0)
 BackwardsCompatibleVersion(version="3.1.0", handler=handle_version_3_1_0)
 BackwardsCompatibleVersion(version="3.2.0", handler=handle_version_3_2_0)
 BackwardsCompatibleVersion(version="3.3.0", handler=handle_version_3_3_0)
-BackwardsCompatibleVersion(version="4.0.0", handler=handle_current_version)
+BackwardsCompatibleVersion(version="4.0.0", handler=handle_version_4_0_0)
+BackwardsCompatibleVersion(version="5.0.1", handler=handle_current_version)
 
 
 def upgrade_metadata(fresh_metadata: dict[str, Any]) -> dict[str, Any]:

{dapla_toolbelt_metadata-0.7.2 → dapla_toolbelt_metadata-0.8.1}/src/dapla_metadata/datasets/utility/utils.py

@@ -43,11 +43,6 @@ DatadocMetadataType = (
 )
 DatasetType = all_optional_model.Dataset | required_model.Dataset
 OptionalDatadocMetadataType = DatadocMetadataType | None
-ExistingPseudonymizationMetadataType = (
-    all_optional_model.PseudonymizationMetadata
-    | required_model.PseudonymizationMetadata
-    | None
-)
 
 
 def get_timestamp_now() -> datetime.datetime:
@@ -114,22 +109,22 @@ def set_default_values_variables(variables: list) -> None:
         variables: A list of variable objects to set default values on.
 
     Example:
-        >>> variables = [model.Variable(short_name="pers",id=None, is_personal_data = None), model.Variable(short_name="fnr",id='9662875c-c245-41de-b667-12ad2091a1ee', is_personal_data='PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA')]
+        >>> variables = [model.Variable(short_name="pers",id=None, is_personal_data = None), model.Variable(short_name="fnr",id='9662875c-c245-41de-b667-12ad2091a1ee', is_personal_data=True)]
         >>> set_default_values_variables(variables)
         >>> isinstance(variables[0].id, uuid.UUID)
         True
 
-        >>> variables[1].is_personal_data == 'PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA'
+        >>> variables[1].is_personal_data == True
         True
 
-        >>> variables[0].is_personal_data == 'NOT_PERSONAL_DATA'
+        >>> variables[0].is_personal_data == False
         True
     """
     for v in variables:
         if v.id is None:
             v.id = uuid.uuid4()
         if v.is_personal_data is None:
-            v.is_personal_data = model.IsPersonalData.NOT_PERSONAL_DATA
+            v.is_personal_data = False
         if v.variable_role is None:
             v.variable_role = VariableRole.MEASURE
 

{dapla_toolbelt_metadata-0.7.2 → dapla_toolbelt_metadata-0.8.1}/src/dapla_metadata/variable_definitions/_utils/config.py

@@ -7,6 +7,7 @@ from dapla_metadata.variable_definitions._generated.vardef_client.configuration
     Configuration,
 )
 
+VARDEF_HOST_PROD = "https://metadata.intern.ssb.no"
 VARDEF_HOST_TEST = "https://metadata.intern.test.ssb.no"
 WORKSPACE_DIR = "WORKSPACE_DIR"
 
@@ -36,8 +37,7 @@ def get_vardef_host() -> str:
     """
     match get_dapla_environment():
         case DaplaEnvironment.PROD:
-            msg = "Vardef is not available in prod."
-            raise NotImplementedError(msg)
+            return VARDEF_HOST_PROD
         case DaplaEnvironment.TEST:
             return VARDEF_HOST_TEST
         case DaplaEnvironment.DEV: