dapla-toolbelt-metadata 0.8.5__py3-none-any.whl → 0.9.1__py3-none-any.whl

dapla_metadata/datasets/code_list.py

@@ -121,7 +121,7 @@ class CodeList(GetExternalSource):
                 classifications_dataframes[i] = (
                     KlassClassification(
                         str(self.classification_id),
-                        str(i),  # type: ignore [arg-type]
+                        i.lower(),  # type: ignore [arg-type]
                     )
                     .get_codes()
                     .data

dapla_metadata/datasets/compatibility/__init__.py

@@ -0,0 +1,10 @@
+"""Model Backwards Compatibility.
+
+This package contains code for upgrading existing metadata documents to the newest version of the model.
+This is analogous to a Database Migration where the structure of the data has changed and we wish to
+retain already persisted information.
+
+"""
+
+from ._utils import is_metadata_in_container_structure
+from .model_backwards_compatibility import upgrade_metadata

dapla_metadata/datasets/compatibility/_handlers.py

@@ -0,0 +1,363 @@
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+
+from dapla_metadata.datasets.compatibility._utils import DATADOC_KEY
+from dapla_metadata.datasets.compatibility._utils import DATASET_KEY
+from dapla_metadata.datasets.compatibility._utils import DOCUMENT_VERSION_KEY
+from dapla_metadata.datasets.compatibility._utils import PSEUDONYMIZATION_KEY
+from dapla_metadata.datasets.compatibility._utils import VARIABLES_KEY
+from dapla_metadata.datasets.compatibility._utils import add_container
+from dapla_metadata.datasets.compatibility._utils import cast_to_date_type
+from dapla_metadata.datasets.compatibility._utils import convert_datetime_to_date
+from dapla_metadata.datasets.compatibility._utils import convert_is_personal_data
+from dapla_metadata.datasets.compatibility._utils import copy_pseudonymization_metadata
+from dapla_metadata.datasets.compatibility._utils import (
+    find_and_update_language_strings,
+)
+from dapla_metadata.datasets.compatibility._utils import remove_element_from_model
+
+
+def handle_current_version(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle the current version of the metadata.
+
+    This function returns the supplied metadata unmodified.
+
+    Args:
+        supplied_metadata: The metadata for the current version.
+
+    Returns:
+        The unmodified supplied metadata.
+    """
+    return supplied_metadata
+
+
+def handle_version_6_0_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 6.1.0.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 6.1.0. Specifically, it:
+    - Consolidates `use_restriction` and `use_restriction_date` into a list of
+      dictionaries under `use_restrictions`.
+    - Removes the old `use_restriction` and `use_restriction_date` fields.
+    - It also converts `use_restriction_date` from datetime to date.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The upgraded metadata dictionary.
+    """
+    dataset = supplied_metadata[DATADOC_KEY][DATASET_KEY]
+
+    use_restriction = dataset.get("use_restriction")
+    if use_restriction is not None:
+        converted_date = convert_datetime_to_date(dataset.get("use_restriction_date"))
+        dataset["use_restrictions"] = [
+            {
+                "use_restriction_type": use_restriction,
+                "use_restriction_date": converted_date,
+            }
+        ]
+    else:
+        dataset["use_restrictions"] = []
+
+    for field in ("use_restriction", "use_restriction_date"):
+        remove_element_from_model(dataset, field)
+
+    return supplied_metadata
+
+
+def handle_version_5_0_1(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 6.0.0.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 6.0.0. Specifically, it:
+    - Moves the following fields from dataset to variable level:
+        - contains_personal_data (becomes is_personal_data)
+        - unit_type
+        - data_source
+        - temporality_type
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+    """
+    fields = [
+        ("contains_personal_data", "is_personal_data"),
+        ("unit_type", "unit_type"),
+        ("data_source", "data_source"),
+        ("temporality_type", "temporality_type"),
+    ]
+
+    dataset: dict[str, Any] = supplied_metadata[DATADOC_KEY][DATASET_KEY]
+    variables: list[dict[str, Any]] = supplied_metadata[DATADOC_KEY][VARIABLES_KEY]
+
+    for f in fields:
+        dataset_level_field_value = dataset.pop(f[0], None)
+        for v in variables:
+            if v.get(f[1]) is None:
+                # Don't override any set values
+                v[f[1]] = dataset_level_field_value
+
+    return supplied_metadata
+
+
+def handle_version_4_0_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 5.0.1.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 5.0.1. Specifically, it:
+    - Copies pseudonymization metadata if pseudonymization is enabled.
+    - Converts the 'is_personal_data' fields to be a bool.
+    - All 'pseudonymization' from the container is removed.
+    - It also updates the container version to 1.0.0 from 0.0.1
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+    """
+    if supplied_metadata.get(PSEUDONYMIZATION_KEY):
+        copy_pseudonymization_metadata(supplied_metadata)
+
+    convert_is_personal_data(supplied_metadata)
+
+    remove_element_from_model(supplied_metadata, PSEUDONYMIZATION_KEY)
+    supplied_metadata[DOCUMENT_VERSION_KEY] = "1.0.0"
+    return supplied_metadata
+
+
+def handle_version_3_3_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 3.3.0.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 4.0.0. Specifically, it removes the
+    'direct_person_identifying' field from each variable in 'datadoc.variables'.
+
+    Version 4.0.0 used an enum for is_personal_data, however this was changed to a bool again for version 5.0.1.
+    We skip setting the enum here and just keep the value it has.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+    """
+    for variable in supplied_metadata[DATADOC_KEY][VARIABLES_KEY]:
+        variable["is_personal_data"] = variable["direct_person_identifying"]
+        remove_element_from_model(variable, "direct_person_identifying")
+
+    return supplied_metadata
+
+
+def handle_version_3_2_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 3.2.0.
+
+    This function modifies the supplied metadata to accommodate breaking
+    changes introduced in version 3.3.0. Specifically, it updates the
+    'contains_data_from' and 'contains_data_until' fields in both the 'dataset'
+    and 'variables' sections of the supplied metadata dictionary to ensure they
+    are stored as date strings.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+    """
+    fields = ["contains_data_from", "contains_data_until"]
+    for field in fields:
+        supplied_metadata[DATADOC_KEY][DATASET_KEY][field] = cast_to_date_type(
+            supplied_metadata[DATADOC_KEY][DATASET_KEY].get(field, None),
+        )
+        for v in supplied_metadata[DATADOC_KEY][VARIABLES_KEY]:
+            v[field] = cast_to_date_type(v.get(field, None))
+
+    return supplied_metadata
+
+
+def handle_version_3_1_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 3.1.0.
+
+    This function modifies the supplied metadata to accommodate breaking
+    changes introduced in version 3.2.0. Specifically, it updates the
+    'data_source' field in both the 'dataset' and 'variables' sections of the
+    supplied metadata dictionary by converting value to string.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+    """
+    data = supplied_metadata[DATADOC_KEY][DATASET_KEY]["data_source"]
+
+    if data is not None:
+        supplied_metadata[DATADOC_KEY][DATASET_KEY]["data_source"] = str(
+            data[0]["languageText"],
+        )
+
+    for i in range(len(supplied_metadata[DATADOC_KEY][VARIABLES_KEY])):
+        data = supplied_metadata[DATADOC_KEY][VARIABLES_KEY][i]["data_source"]
+        if data is not None:
+            supplied_metadata[DATADOC_KEY][VARIABLES_KEY][i]["data_source"] = str(
+                data[0]["languageText"],
+            )
+
+    return supplied_metadata
+
+
+def handle_version_2_2_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 2.2.0.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 3.1.0. Specifically, it updates the 'subject_field' in
+    the 'dataset' section of the supplied metadata dictionary by converting it to
+    a string. It also removes the 'register_uri' field from the 'dataset'.
+    Additionally, it removes 'sentinel_value_uri' from each variable,
+    sets 'special_value' and 'custom_type' fields to None, and updates
+    language strings in the 'variables' and 'dataset' sections.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+    """
+    if supplied_metadata[DATADOC_KEY][DATASET_KEY]["subject_field"] is not None:
+        data = supplied_metadata[DATADOC_KEY][DATASET_KEY]["subject_field"]
+        supplied_metadata[DATADOC_KEY][DATASET_KEY]["subject_field"] = str(
+            data["nb"] or data["nn"] or data["en"],
+        )
+
+    remove_element_from_model(
+        supplied_metadata[DATADOC_KEY][DATASET_KEY], "register_uri"
+    )
+
+    for i in range(len(supplied_metadata[DATADOC_KEY][VARIABLES_KEY])):
+        remove_element_from_model(
+            supplied_metadata[DATADOC_KEY][VARIABLES_KEY][i],
+            "sentinel_value_uri",
+        )
+        supplied_metadata[DATADOC_KEY][VARIABLES_KEY][i]["special_value"] = None
+        supplied_metadata[DATADOC_KEY][VARIABLES_KEY][i]["custom_type"] = None
+        supplied_metadata[DATADOC_KEY][VARIABLES_KEY][i] = (
+            find_and_update_language_strings(
+                supplied_metadata[DATADOC_KEY][VARIABLES_KEY][i],
+            )
+        )
+    supplied_metadata[DATADOC_KEY][DATASET_KEY]["custom_type"] = None
+    supplied_metadata[DATADOC_KEY][DATASET_KEY] = find_and_update_language_strings(
+        supplied_metadata[DATADOC_KEY][DATASET_KEY],
+    )
+    return supplied_metadata
+
+
+def handle_version_2_1_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 2.1.0.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 2.2.0. Specifically, it updates the 'owner' field in
+    the 'dataset' section of the supplied metadata dictionary by converting it
+    from a LanguageStringType to a string.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+    """
+    data = supplied_metadata[DATASET_KEY]["owner"]
+    supplied_metadata[DATASET_KEY]["owner"] = str(
+        data["nb"] or data["nn"] or data["en"]
+    )
+    return add_container(supplied_metadata)
+
+
+def handle_version_1_0_0(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 1.0.0.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 2.1.0. Specifically, it updates the date fields
+    'metadata_created_date' and 'metadata_last_updated_date' to ISO 8601 format
+    with UTC timezone. It also converts the 'data_source' field from a string to a
+    dictionary with language keys if necessary and removes the 'data_source_path'
+    field.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+
+    """
+    datetime_fields = [("metadata_created_date"), ("metadata_last_updated_date")]
+    for field in datetime_fields:
+        if supplied_metadata[DATASET_KEY][field]:
+            supplied_metadata[DATASET_KEY][field] = datetime.isoformat(
+                datetime.fromisoformat(
+                    supplied_metadata[DATASET_KEY][field]
+                ).astimezone(
+                    tz=timezone.utc,
+                ),
+                timespec="seconds",
+            )
+    if isinstance(supplied_metadata[DATASET_KEY]["data_source"], str):
+        supplied_metadata[DATASET_KEY]["data_source"] = {
+            "en": supplied_metadata[DATASET_KEY]["data_source"],
+            "nn": "",
+            "nb": "",
+        }
+
+    remove_element_from_model(supplied_metadata[DATASET_KEY], "data_source_path")
+
+    return supplied_metadata
+
+
+def handle_version_0_1_1(supplied_metadata: dict[str, Any]) -> dict[str, Any]:
+    """Handle breaking changes for version 0.1.1.
+
+    This function modifies the supplied metadata to accommodate breaking changes
+    introduced in version 1.0.0. Specifically, it renames certain keys within the
+    `dataset` and `variables` sections, and replaces empty string values with
+    `None` for `dataset` keys.
+
+    Args:
+        supplied_metadata: The metadata dictionary that needs to be updated.
+
+    Returns:
+        The updated metadata dictionary.
+
+    References:
+        PR ref: https://github.com/statisticsnorway/ssb-datadoc-model/pull/4
+    """
+    key_renaming = [
+        ("metadata_created_date", "created_date"),
+        ("metadata_created_by", "created_by"),
+        ("metadata_last_updated_date", "last_updated_date"),
+        ("metadata_last_updated_by", "last_updated_by"),
+    ]
+    for new_key, old_key in key_renaming:
+        supplied_metadata[DATASET_KEY][new_key] = supplied_metadata[DATASET_KEY].pop(
+            old_key,
+        )
+    # Replace empty strings with None, empty strings are not valid for LanguageStrings values
+    supplied_metadata[DATASET_KEY] = {
+        k: None if v == "" else v for k, v in supplied_metadata[DATASET_KEY].items()
+    }
+
+    key_renaming = [("data_type", "datatype")]
+
+    for i in range(len(supplied_metadata[VARIABLES_KEY])):
+        for new_key, old_key in key_renaming:
+            supplied_metadata[VARIABLES_KEY][i][new_key] = supplied_metadata[
+                VARIABLES_KEY
+            ][i].pop(
+                old_key,
+            )
+
+    return supplied_metadata

dapla_metadata/datasets/compatibility/_utils.py

@@ -0,0 +1,259 @@
+from datetime import datetime
+from typing import Any
+
+import arrow
+
+DOCUMENT_VERSION_KEY = "document_version"
+DATADOC_KEY = "datadoc"
+DATASET_KEY = "dataset"
+VARIABLES_KEY = "variables"
+PSEUDONYMIZATION_KEY = "pseudonymization"
+
+
+class UnknownModelVersionError(Exception):
+    """Exception raised for unknown model versions.
+
+    This error is thrown when an unrecognized model version is encountered.
+    """
+
+    def __init__(
+        self,
+        supplied_version: str,
+        *args: tuple[Any, ...],
+    ) -> None:
+        """Initialize the exception with the supplied version.
+
+        Args:
+            supplied_version: The version of the model that was not recognized.
+            *args: Additional arguments for the Exception base class.
+        """
+        super().__init__(args)
+        self.supplied_version = supplied_version
+
+    def __str__(self) -> str:
+        """Return string representation."""
+        return f"Document Version ({self.supplied_version}) of discovered file is not supported"
+
+
+def _convert_language_string_type(supplied_value: dict) -> list[dict[str, str]]:
+    """Convert a dictionary of language-specific strings to a list of dictionaries.
+
+    This function takes a dictionary with language codes as keys and
+    corresponding language-specific strings as values, and converts it to a list
+    of dictionaries with 'languageCode' and 'languageText' keys.
+
+    Args:
+        supplied_value: A dictionary containing language codes as keys and
+            language strings as values.
+
+    Returns:
+        A list of dictionaries, each containing 'languageCode' and 'languageText'
+        keys, representing the converted language strings.
+    """
+    return [
+        {
+            "languageCode": "en",
+            "languageText": supplied_value["en"],
+        },
+        {
+            "languageCode": "nn",
+            "languageText": supplied_value["nn"],
+        },
+        {
+            "languageCode": "nb",
+            "languageText": supplied_value["nb"],
+        },
+    ]
+
+
+def find_and_update_language_strings(supplied_metadata: dict | None) -> dict | None:
+    """Find and update language-specific strings in the supplied metadata.
+
+    This function iterates through the supplied metadata dictionary.
+    For each key-value pair, if the value is a dictionary containing "en"
+    it is passed to the `_convert_language_string_type` function to potentially
+    update its format.
+
+    Args:
+        supplied_metadata: A metadata dictionary where values may include nested
+            dictionaries with language-specific strings.
+
+    Returns:
+        The updated metadata dictionary. If the supplied metadata is not a
+        dictionary, it returns `None`.
+    """
+    if isinstance(supplied_metadata, dict):
+        for key, value in supplied_metadata.items():
+            if isinstance(value, dict) and "en" in value:
+                supplied_metadata[key] = _convert_language_string_type(value)
+        return supplied_metadata
+    return None
+
+
+def remove_element_from_model(
+    supplied_metadata: dict[str, Any],
+    element_to_remove: str,
+) -> None:
+    """Remove an element from the supplied metadata dictionary.
+
+    This function deletes a specified element from the supplied metadata dictionary
+    if it exists.
+
+    Args:
+        supplied_metadata: The metadata dictionary from which the element will be
+            removed.
+        element_to_remove: The key of the element to be removed from the metadata
+            dictionary.
+    """
+    supplied_metadata.pop(element_to_remove, None)
+
+
+def cast_to_date_type(value_to_update: str | None) -> str | None:
+    """Convert a string to a date string in ISO format.
+
+    This function takes a string representing a date and converts it to a
+    date string in ISO format. If the input is `None`, it returns `None` without
+    modification.
+
+    Args:
+        value_to_update: A string representing a date or `None`.
+
+    Returns:
+        The date string in ISO format if the input was a valid date string, or
+        `None` if the input was `None`.
+    """
+    if value_to_update is None:
+        return value_to_update
+
+    return str(
+        arrow.get(
+            value_to_update,
+        ).date(),
+    )
+
+
+def convert_is_personal_data(supplied_metadata: dict[str, Any]) -> None:
+    """Convert 'is_personal_data' values in the supplied metadata to boolean.
+
+    Iterates over variables in the supplied metadata and updates the
+    'is_personal_data' field:
+      - Sets it to True for NON_PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA and PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA.
+      - Sets it to False for NOT_PERSONAL_DATA.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+    """
+    for variable in supplied_metadata[DATADOC_KEY][VARIABLES_KEY]:
+        value = variable["is_personal_data"]
+        if value in (
+            "NON_PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA",
+            "PSEUDONYMISED_ENCRYPTED_PERSONAL_DATA",
+        ):
+            variable["is_personal_data"] = True
+        elif value == "NOT_PERSONAL_DATA":
+            variable["is_personal_data"] = False
+
+
+def copy_pseudonymization_metadata(supplied_metadata: dict[str, Any]) -> None:
+    """Copies pseudonymization metadata from the old pseudonymization section into the corresponding variable.
+
+    For each variable in `supplied_metadata[DATADOC_KEY][VARIABLES_KEY]` that has a matching
+    `short_name` in `supplied_metadata[PSEUDONYMIZATION_KEY]["pseudo_variables"]`, this
+    function copies the following fields into the variable's 'pseudonymization' dictionary:
+
+        - stable_identifier_type
+        - stable_identifier_version
+        - encryption_algorithm
+        - encryption_key_reference
+        - encryption_algorithm_parameters
+
+    From the pseudo_dataset the value dataset_pseudo_time is copied to each variable as pseudonymization_time.
+
+    Args:
+        supplied_metadata: The metadata dictionary to be updated.
+    """
+    pseudo_vars = supplied_metadata.get(PSEUDONYMIZATION_KEY, {}).get(
+        "pseudo_variables", []
+    )
+    pseudo_dataset = (
+        supplied_metadata.get(PSEUDONYMIZATION_KEY, {}).get("pseudo_dataset") or {}
+    )
+    pseudo_time = pseudo_dataset.get("dataset_pseudo_time", None)
+    datadoc_vars = supplied_metadata.get(DATADOC_KEY, {}).get(VARIABLES_KEY, [])
+    pseudo_lookup = {var.get("short_name"): var for var in pseudo_vars}
+
+    for variable in datadoc_vars:
+        short_name = variable.get("short_name")
+        if short_name in pseudo_lookup:
+            pseudo_var = pseudo_lookup[short_name]
+            variable[PSEUDONYMIZATION_KEY] = variable.get(
+                PSEUDONYMIZATION_KEY, {}
+            ).copy()
+
+            for field in [
+                "stable_identifier_type",
+                "stable_identifier_version",
+                "encryption_algorithm",
+                "encryption_key_reference",
+                "encryption_algorithm_parameters",
+            ]:
+                variable[PSEUDONYMIZATION_KEY][field] = pseudo_var[field]
+            variable[PSEUDONYMIZATION_KEY]["pseudonymization_time"] = pseudo_time
+
+        else:
+            variable[PSEUDONYMIZATION_KEY] = None
+
+
+def convert_datetime_to_date(date_value: str | None) -> str | None:
+    """Convert ISO datetime string to date string, handling None and invalid values."""
+    if not date_value or not isinstance(date_value, str):
+        return date_value
+
+    try:
+        dt = datetime.fromisoformat(date_value.replace("Z", "+00:00"))
+        return dt.date().isoformat()
+    except ValueError:
+        return date_value
+
+
+def add_container(existing_metadata: dict) -> dict:
+    """Add container for previous versions.
+
+    Adds a container structure for previous versions of metadata.
+    This function wraps the existing metadata in a new container structure
+    that includes the 'document_version', 'datadoc', and 'pseudonymization'
+    fields. The 'document_version' is set to "0.0.1" and 'pseudonymization'
+    is set to None.
+
+    Args:
+        existing_metadata: The original metadata dictionary to be wrapped.
+
+    Returns:
+        A new dictionary containing the wrapped metadata with additional fields.
+    """
+    return {
+        DOCUMENT_VERSION_KEY: "0.0.1",
+        DATADOC_KEY: existing_metadata,
+        PSEUDONYMIZATION_KEY: None,
+    }
+
+
+def is_metadata_in_container_structure(
+    metadata: dict,
+) -> bool:
+    """Check if the metadata is in the container structure.
+
+    At a certain point a metadata 'container' was introduced.
+    The container provides a structure for different 'types' of metadata, such as
+    'datadoc', 'pseudonymization' etc.
+    This function determines if the given metadata dictionary follows this container
+    structure by checking for the presence of the 'datadoc' field.
+
+    Args:
+        metadata: The metadata dictionary to check.
+
+    Returns:
+        True if the metadata is in the container structure (i.e., contains the
+        'datadoc' field), False otherwise.
+    """
+    return DATADOC_KEY in metadata