dapla-toolbelt-metadata 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

dapla_metadata/datasets/core.py

@@ -109,8 +109,11 @@ class Datadoc:
         self.dataset_path: pathlib.Path | CloudPath | None = None
         self.dataset = model.Dataset()
         self.variables: list = []
+        self.pseudo_variables: list[model.PseudoVariable] = []
         self.variables_lookup: dict[str, model.Variable] = {}
+        self.pseudo_variables_lookup: dict[str, model.PseudoVariable] = {}
         self.explicitly_defined_metadata_document = False
+        self.dataset_consistency_status: list = []
         if metadata_document_path:
             self.metadata_document = normalize_path(metadata_document_path)
             self.explicitly_defined_metadata_document = True
@@ -148,11 +151,19 @@ class Datadoc:
         """
         extracted_metadata: model.DatadocMetadata | None = None
         existing_metadata: model.DatadocMetadata | None = None
+        existing_pseudonymization: model.PseudonymizationMetadata | None = None
+
         if self.metadata_document and self.metadata_document.exists():
             existing_metadata = self._extract_metadata_from_existing_document(
                 self.metadata_document,
             )
 
+            existing_pseudonymization = (
+                self._extract_pseudonymization_from_existing_document(
+                    self.metadata_document,
+                )
+            )
+
         if (
             self.dataset_path is not None
             and self.dataset == model.Dataset()
@@ -169,11 +180,14 @@ class Datadoc:
             and existing_metadata is not None
         ):
             existing_file_path = self._get_existing_file_path(extracted_metadata)
-            self._check_ready_to_merge(
+            self.dataset_consistency_status = self._check_dataset_consistency(
                 self.dataset_path,
                 Path(existing_file_path),
                 extracted_metadata,
                 existing_metadata,
+            )
+            self._check_ready_to_merge(
+                self.dataset_consistency_status,
                 errors_as_warnings=self.errors_as_warnings,
             )
             merged_metadata = self._merge_metadata(
@@ -188,10 +202,15 @@ class Datadoc:
             self._set_metadata(merged_metadata)
         else:
             self._set_metadata(existing_metadata or extracted_metadata)
+
+        if existing_pseudonymization:
+            self._set_pseudonymization_metadata(existing_pseudonymization)
+
         set_default_values_variables(self.variables)
         set_default_values_dataset(self.dataset)
         set_dataset_owner(self.dataset)
         self._create_variables_lookup()
+        self._create_pseudo_variables_lookup()
 
     def _get_existing_file_path(
         self,
@@ -218,35 +237,48 @@ class Datadoc:
         self.dataset = merged_metadata.dataset
         self.variables = merged_metadata.variables
 
+    def _set_pseudonymization_metadata(
+        self,
+        existing_pseudonymization: model.PseudonymizationMetadata | None,
+    ) -> None:
+        if not existing_pseudonymization or not (
+            existing_pseudonymization.pseudo_variables
+        ):
+            msg = "Could not read pseudonymization metadata"
+            raise ValueError(msg)
+        self.pseudo_variables = existing_pseudonymization.pseudo_variables
+
     def _create_variables_lookup(self) -> None:
         self.variables_lookup = {
             v.short_name: v for v in self.variables if v.short_name
         }
 
+    def _create_pseudo_variables_lookup(self) -> None:
+        self.pseudo_variables_lookup = {
+            v.short_name: v for v in self.pseudo_variables if v.short_name
+        }
+
     @staticmethod
-    def _check_ready_to_merge(
+    def _check_dataset_consistency(
         new_dataset_path: Path | CloudPath,
         existing_dataset_path: Path,
         extracted_metadata: model.DatadocMetadata,
         existing_metadata: model.DatadocMetadata,
-        *,
-        errors_as_warnings: bool,
-    ) -> None:
-        """Check if the datasets are consistent enough to make a successful merge of metadata.
+    ) -> list[dict[str, object]]:
+        """Run consistency tests.
 
         Args:
             new_dataset_path: Path to the dataset to be documented.
             existing_dataset_path: Path stored in the existing metadata.
             extracted_metadata: Metadata extracted from a physical dataset.
             existing_metadata: Metadata from a previously created metadata document.
-            errors_as_warnings: True if failing checks should be raised as warnings, not errors.
 
-        Raises:
-            InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+        Returns:
+            List if dict with property name and boolean success flag
         """
         new_dataset_path_info = DaplaDatasetPathInfo(new_dataset_path)
         existing_dataset_path_info = DaplaDatasetPathInfo(existing_dataset_path)
-        results = [
+        return [
             {
                 "name": "Bucket name",
                 "success": (
@@ -290,6 +322,20 @@ class Datadoc:
                 ),
             },
         ]
+
+    @staticmethod
+    def _check_ready_to_merge(
+        results: list[dict[str, object]], *, errors_as_warnings: bool
+    ) -> None:
+        """Check if the datasets are consistent enough to make a successful merge of metadata.
+
+        Args:
+            results: List if dict with property name and boolean success flag
+            errors_as_warnings: True if failing checks should be raised as warnings, not errors.
+
+        Raises:
+            InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+        """
         if failures := [result for result in results if not result["success"]]:
             msg = f"{INCONSISTENCIES_MESSAGE} {', '.join(str(f['name']) for f in failures)}"
             if errors_as_warnings:
@@ -384,6 +430,42 @@ class Datadoc:
             )
             return None
 
+    def _extract_pseudonymization_from_existing_document(
+        self,
+        document: pathlib.Path | CloudPath,
+    ) -> model.PseudonymizationMetadata | None:
+        """Read pseudo metadata from an existing metadata document.
+
+        If there is pseudo metadata in the document supplied, the method validates and returns the pseudonymization structure.
+
+        Args:
+            document: A path to the existing metadata document.
+
+        Raises:
+            json.JSONDecodeError: If the metadata document cannot be parsed.
+        """
+        try:
+            with document.open(mode="r", encoding="utf-8") as file:
+                fresh_metadata = json.load(file)
+        except json.JSONDecodeError:
+            logger.warning(
+                "Could not open existing metadata file %s.",
+                document,
+                exc_info=True,
+            )
+            return None
+
+        if not is_metadata_in_container_structure(fresh_metadata):
+            return None
+
+        pseudonymization_metadata = fresh_metadata.get("pseudonymization")
+        if pseudonymization_metadata is None:
+            return None
+
+        return model.PseudonymizationMetadata.model_validate_json(
+            json.dumps(pseudonymization_metadata),
+        )
+
     def _extract_subject_field_from_path(
         self,
         dapla_dataset_path_info: DaplaDatasetPathInfo,
@@ -501,6 +583,11 @@ class Datadoc:
         )
         if self.container:
             self.container.datadoc = datadoc
+            if not self.container.pseudonymization:
+                self.container.pseudonymization = model.PseudonymizationMetadata(
+                    pseudo_dataset=model.PseudoDataset()
+                )
+            self.container.pseudonymization.pseudo_variables = self.pseudo_variables
         else:
             self.container = model.MetadataContainer(datadoc=datadoc)
         if self.metadata_document:
@@ -530,3 +617,16 @@ class Datadoc:
             self.dataset,
         ) + num_obligatory_variables_fields_completed(self.variables)
         return calculate_percentage(num_set_fields, num_all_fields)
+
+    def add_pseudo_variable(self, variable_short_name: str) -> None:
+        """Adds a new pseudo variable to the list of pseudonymized variables."""
+        if self.variables_lookup[variable_short_name] is not None:
+            pseudo_variable = model.PseudoVariable(short_name=variable_short_name)
+            self.pseudo_variables.append(pseudo_variable)
+            self.pseudo_variables_lookup[variable_short_name] = pseudo_variable
+
+    def get_pseudo_variable(
+        self, variable_short_name: str
+    ) -> model.PseudoVariable | None:
+        """Finds a pseudo variable by shortname."""
+        return self.pseudo_variables_lookup.get(variable_short_name)

dapla_metadata/standards/name_validator.py

@@ -128,9 +128,9 @@ class NamingStandardReport:
         """Returns an appropriate message based on the success rate."""
         rate = self.success_rate()
         if rate is not None:
-            if rate == 100:
+            if 95 <= rate <= 100:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_BEST
-            if 70 < rate < 100:
+            if 70 < rate < 95:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_GOOD
             if 40 <= rate <= 70:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_AVERAGE

dapla_metadata/standards/utils/constants.py

@@ -9,7 +9,7 @@ NAME_STANDARD_SUCCESS = "Filene dine er i samsvar med SSB-navnestandarden"
 NAME_STANDARD_VIOLATION = "Det er oppdaget brudd på SSB-navnestandard:"
 
 MISSING_BUCKET_NAME = "Filnavn mangler bøttenavn ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
-MISSING_VERSION = "Filnavn mangler versjon ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
+MISSING_VERSION = "Filnavn mangler versjon, hvis ikke filen er nyeste versjon kan dette være brudd på navnestandarden ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#versjonering-av-datasett"
 MISSING_PERIOD = "Filnavn mangler gyldighetsperiode ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
 MISSING_SHORT_NAME = "Kortnavn for statistikk mangler ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
 MISSING_DATA_STATE = "Mappe for datatilstand mangler ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
@@ -26,7 +26,7 @@ BUCKET_NAME_UNKNOWN = "Kan ikke validere bøttenavn"
 
 SSB_NAMING_STANDARD_REPORT = "SSB navnestandard rapport"
 SSB_NAMING_STANDARD_REPORT_SUCCESS_RATE = "Suksess rate"
-SSB_NAMING_STANDARD_REPORT_RESULT_BEST = "🚀 Fantastisk! Alt bestått! 🎉\n"
+SSB_NAMING_STANDARD_REPORT_RESULT_BEST = "🚀 Fantastisk! 🎉\n"
 SSB_NAMING_STANDARD_REPORT_RESULT_GOOD = (
     "✅ Bra jobba! Fortsatt litt rom for forbedring. 😊\n"
 )

dapla_metadata/variable_definitions/_utils/config.py

@@ -59,8 +59,7 @@ def get_vardef_host() -> str:
         case DaplaEnvironment.TEST:
             return VARDEF_HOST_TEST
         case DaplaEnvironment.DEV:
-            msg = "Vardef is not available in dev."
-            raise NotImplementedError(msg)
+            return VARDEF_HOST_TEST
         case _:
             return get_config_item("VARDEF_HOST") or "http://localhost:8080"
 

dapla_metadata/variable_definitions/_utils/constants.py

@@ -39,3 +39,26 @@ MACHINE_GENERATED_FIELDS = [
 
 OPTIONAL_FIELD = "~ Valgfritt felt ~"
 REQUIRED_FIELD = "! Obligatorisk felt !"
+
+YAML_STR_TAG = "tag:yaml.org,2002:str"
+
+BLOCK_FIELDS = [
+    "definition",
+    "name",
+    "contact.title",
+    "comment",
+]
+
+DOUBLE_QUOTE_FIELDS = [
+    "unit_types",
+    "subject_fields",
+    "related_variable_definition_uris",
+    "owner",
+    "short_name",
+    "classification_reference",
+    "measurement_type",
+    "external_reference_uri",
+    "created_by",
+    "id",
+    "last_updated_by",
+]

dapla_metadata/variable_definitions/_utils/files.py

@@ -4,12 +4,16 @@ import logging
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING
+from typing import Any
 from typing import cast
 
 import pytz
 from pydantic.config import JsonDict
 from ruamel.yaml import YAML
 from ruamel.yaml import CommentedMap
+from ruamel.yaml import RoundTripRepresenter
+from ruamel.yaml.scalarstring import DoubleQuotedScalarString
+from ruamel.yaml.scalarstring import LiteralScalarString
 
 from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
     CompleteResponse,
@@ -18,6 +22,8 @@ from dapla_metadata.variable_definitions._generated.vardef_client.models.variabl
     VariableStatus,
 )
 from dapla_metadata.variable_definitions._utils import config
+from dapla_metadata.variable_definitions._utils.constants import BLOCK_FIELDS
+from dapla_metadata.variable_definitions._utils.constants import DOUBLE_QUOTE_FIELDS
 from dapla_metadata.variable_definitions._utils.constants import (
     MACHINE_GENERATED_FIELDS,
 )
@@ -40,6 +46,7 @@ from dapla_metadata.variable_definitions._utils.constants import (
 from dapla_metadata.variable_definitions._utils.constants import (
     VARIABLE_STATUS_FIELD_NAME,
 )
+from dapla_metadata.variable_definitions._utils.constants import YAML_STR_TAG
 from dapla_metadata.variable_definitions._utils.descriptions import (
     apply_norwegian_descriptions_to_model,
 )
@@ -112,6 +119,15 @@ def _get_variable_definitions_dir():
     return folder_path
 
 
+def _set_field_requirement(field_name: str, field: Any) -> str | None:
+    """Determine the field requirement status."""
+    if field_name not in MACHINE_GENERATED_FIELDS:
+        if field.is_required() or field_name == VARIABLE_STATUS_FIELD_NAME:
+            return REQUIRED_FIELD
+        return OPTIONAL_FIELD
+    return None
+
+
 def _populate_commented_map(
     field_name: str,
     value: str,
@@ -120,16 +136,17 @@ def _populate_commented_map(
 ) -> None:
     """Add data to a CommentedMap."""
     commented_map[field_name] = value
-    field = model_instance.model_fields[field_name]
+    field = type(model_instance).model_fields[field_name]
     description: JsonValue = cast(
         JsonDict,
         field.json_schema_extra,
     )[NORWEGIAN_DESCRIPTIONS]
+    field_requirement: str | None = _set_field_requirement(field_name, field)
     if description is not None:
         new_description = (
-            (REQUIRED_FIELD if field.is_required() else OPTIONAL_FIELD)
-            + "\n"
-            + str(description)
+            ("\n" + field_requirement + "\n" + str(description))
+            if field_requirement
+            else ("\n" + str(description))
         )
         commented_map.yaml_set_comment_before_after_key(
             field_name,
@@ -174,14 +191,22 @@ def _validate_and_create_directory(custom_directory: Path) -> Path:
     return custom_directory
 
 
-def _configure_yaml() -> YAML:
-    yaml = YAML()  # Use ruamel.yaml library
-    yaml.default_flow_style = False  # Ensures pretty YAML formatting
-
+def configure_yaml(yaml: YAML) -> YAML:
+    """Common Yaml config for variable definitions."""
+    yaml.Representer = RoundTripRepresenter  # Preserve the order of keys etc.
+    yaml.default_flow_style = False  # Ensures pretty YAML formatting block style
+    yaml.allow_unicode = True  # Support special characters
+    yaml.preserve_quotes = True
+    yaml.width = 4096  # prevent wrapping lines
+    yaml.indent(
+        mapping=4,
+        sequence=6,
+        offset=4,
+    )  # Ensure indentation for nested keys and lists
     yaml.representer.add_representer(
         VariableStatus,
         lambda dumper, data: dumper.represent_scalar(
-            "tag:yaml.org,2002:str",
+            YAML_STR_TAG,
             data.value,
         ),
     )
@@ -189,6 +214,76 @@ def _configure_yaml() -> YAML:
     return yaml
 
 
+def _safe_get(data: dict, keys: list):
+    """Safely navigate nested dictionaries."""
+    for key in keys:
+        if not isinstance(data, dict) or key not in data or data[key] is None:
+            return None
+        data = data[key]
+    return data
+
+
+def _apply_literal_scalars(field: dict):
+    """Helper function to wrap `LanguageStringType` values in `LiteralScalarString`.
+
+    This function wraps each non-`None` language value in a `LanguageStringType` field
+    in the `LiteralScalarString` YAML type, ensuring proper YAML formatting with block style.
+    """
+    for lang, value in field.items():
+        if value is not None:
+            field[lang] = LiteralScalarString(value)
+
+
+def _apply_double_quotes_to_dict_values(field: dict):
+    """Helper function to wrap dictionary values in `DoubleQuotedScalarString`.
+
+    This function wraps each non-`None` value in a dictionary, including values inside lists,
+    in the `DoubleQuotedScalarString` YAML type, ensuring proper YAML formatting with double quotes.
+    """
+    for sub_key, sub_value in field.items():
+        if isinstance(sub_value, list):
+            field[sub_key] = [
+                DoubleQuotedScalarString(item) for item in sub_value if item is not None
+            ]
+        elif sub_value is not None:
+            field[sub_key] = DoubleQuotedScalarString(sub_value)
+
+
+def pre_process_data(data: dict) -> dict:
+    """Format variable definition model fields with ruamel YAML scalar string types.
+
+    This method sets the appropriate scalar string type (either `LiteralScalarString` or `DoubleQuotedScalarString`)
+    for fields of the variable definition model, based on predefined lists of fields.
+
+    It processes both nested dictionaries and lists, ensuring each element is formatted with the correct YAML string type.
+
+    Args:
+        data (dict): A dictionary containing the variable definition data.
+
+    Returns:
+        dict: The updated dictionary with model fields formatted as ruamel.yaml scalar string types.
+    """
+    for key in BLOCK_FIELDS:
+        keys = key.split(".")
+        field = _safe_get(data, keys)
+        if isinstance(field, dict):
+            _apply_literal_scalars(field)
+
+    for key in DOUBLE_QUOTE_FIELDS:
+        keys = key.split(".")
+        field = _safe_get(data, keys)
+        if isinstance(field, list):
+            data[key] = [
+                DoubleQuotedScalarString(item) for item in field if item is not None
+            ]
+        elif isinstance(field, str):
+            data[key] = DoubleQuotedScalarString(data[key])
+        elif isinstance(field, dict):
+            _apply_double_quotes_to_dict_values(field)
+
+    return data
+
+
 def _model_to_yaml_with_comments(
     model_instance: CompleteResponse,
     file_name: str,
@@ -209,7 +304,8 @@ def _model_to_yaml_with_comments(
     Returns:
         Path: The file path of the generated YAML file.
     """
-    yaml = _configure_yaml()
+    yaml = YAML()
+    configure_yaml(yaml)
 
     from dapla_metadata.variable_definitions.variable_definition import (
         VariableDefinition,
@@ -223,7 +319,7 @@ def _model_to_yaml_with_comments(
         serialize_as_any=True,
         warnings="error",
     )
-
+    data = pre_process_data(data)
     # One CommentMap for each section in the yaml file
     machine_generated_map = CommentedMap()
     commented_map = CommentedMap()

dapla_metadata/variable_definitions/_utils/template_files.py

@@ -41,31 +41,24 @@ def _get_default_template() -> "VariableDefinition":
 
     return VariableDefinition(
         name=LanguageStringType(
-            nb="default navn",
-            nn="default namn",
-            en="default name",
+            nb="Navn",
         ),
-        short_name="default_kortnavn",
+        short_name="generert_kortnavn",
         definition=LanguageStringType(
-            nb="default definisjon",
-            nn="default definisjon",
-            en="default definition",
+            nb="Definisjonstekst",
         ),
-        classification_reference="class_id",
         valid_from=DEFAULT_DATE,
-        unit_types=["00"],
-        subject_fields=["aa"],
+        unit_types=[""],
+        subject_fields=[""],
         contains_special_categories_of_personal_data=False,
-        variable_status=VariableStatus.DRAFT.value,
         owner=Owner(team="default team", groups=["default group"]),
         contact=Contact(
             title=LanguageStringType(
-                nb="default tittel",
-                nn="default tittel",
-                en="default title",
+                nb="generert tittel",
             ),
-            email="default@ssb.no",
+            email="generert@ssb.no",
         ),
+        variable_status=VariableStatus.DRAFT.value,
         id="",
         patch_id=0,
         created_at=DEFAULT_DATE,

dapla_metadata/variable_definitions/_utils/variable_definition_files.py

@@ -3,6 +3,7 @@
 import logging
 from os import PathLike
 from pathlib import Path
+from typing import Any
 from typing import TypeVar
 
 from pydantic import BaseModel
@@ -17,6 +18,7 @@ from dapla_metadata.variable_definitions._utils.files import _get_current_time
 from dapla_metadata.variable_definitions._utils.files import (
     _model_to_yaml_with_comments,
 )
+from dapla_metadata.variable_definitions._utils.files import configure_yaml
 
 logger = logging.getLogger(__name__)
 
@@ -46,13 +48,38 @@ def create_variable_yaml(
 
 def _read_variable_definition_file(file_path: Path) -> dict:
     yaml = YAML()
-
+    configure_yaml(yaml)
     logger.debug("Full path to variable definition file %s", file_path)
     logger.info("Reading from '%s'", file_path.name)
     with file_path.open(encoding="utf-8") as f:
         return yaml.load(f)
 
 
+def _strip_strings_recursively(data: Any) -> Any:
+    """Recursively strip leading and trailing whitespace from string values in nested dicts/lists.
+
+    This function traverses the provided data, which may be a dictionary, list, or other types,
+    and applies the following logic:
+        - If the data is a dictionary, it recursively strips string values in all key-value pairs.
+        - If the data is a list, it recursively strips string values in all list elements.
+        - If the data is a string, it strips leading and trailing whitespace.
+        - Any other data types are returned unchanged.
+
+    Args:
+        data: The input data, which may include nested dictionaries, lists, or other types.
+
+    Returns:
+        Any: The processed data, with strings stripped of whitespace or unchanged if not a string.
+    """
+    if isinstance(data, dict):
+        return {k: _strip_strings_recursively(v) for k, v in data.items()}
+    if isinstance(data, list):
+        return [_strip_strings_recursively(item) for item in data]
+    if isinstance(data, str):
+        return data.strip()
+    return data
+
+
 def _read_file_to_model(
     file_path: PathLike[str] | None,
     model_class: type[T],
@@ -80,14 +107,14 @@ def _read_file_to_model(
         raise FileNotFoundError(
             msg,
         ) from e
+    raw_data = _read_variable_definition_file(file_path)
+    cleaned_data = _strip_strings_recursively(raw_data)
+
     model = model_class.from_dict(  # type:ignore [attr-defined]
-        _read_variable_definition_file(
-            file_path,
-        ),
+        cleaned_data
     )
 
     if model is None:
         msg = f"Could not read data from {file_path}"
         raise FileNotFoundError(msg)
-
     return model

dapla_metadata/variable_definitions/resources/vardef_model_descriptions_nb.yaml

@@ -1,50 +1,86 @@
 # --- Variabel definisjoner ---
 # ref: https://statistics-norway.atlassian.net/wiki/spaces/MPD/pages/3009839199/VarDef+-+Krav+til+dokumentasjon+av+variabler
 name: |
-  Variabelens navn. Dette skal ikke være en mer “teknisk” forkortelse, men et navn som er forståelig for mennesker, f.eks. “Lønnsinntekter”.
+  Variabelens navn. Dette skal ikke være en mer “teknisk” forkortelse, men et navn som er forståelig for mennesker.
+  -------------------------
+  >>> EKSEMPEL:
+  name:
+    nb: |-
+        Lønnsinntekter
 short_name: |
   Dette er variabelens kortnavn, som kan være en mer “teknisk” forkortelse, f.eks. wlonn (kortnavnet til Lønnsinntekter). Kortnavnet til en variabel i Vardef skal være unikt.
-  Kravet til kortnavnet er at det kan inneholde a-z (kun små bokstaver), 0-9 og _ (understrek). Minimumslengden på kortnavnet er 2 tegn.  Bokstavene “æ”, “ø” og “å” kan ikke brukes. Disse anbefales erstattet med hhv. “ae”, “oe” og “aa"
+  Kravet til kortnavnet er at det kan inneholde a-z (kun små bokstaver), 0-9 og _ (understrek). Minimumslengden på kortnavnet er 2 tegn.
+  Bokstavene “æ”, “ø” og “å” kan ikke brukes. Disse anbefales erstattet med hhv. “ae”, “oe” og “aa"
 definition: |
   En definisjon skal beskrive hva variabelen betyr og være så kort og presis som mulig. Mer utfyllende opplysninger kan legges i Merknad-feltet.
+  -------------------------
+  >>> EKSEMPEL:
+  definition:
+    nb: |-
+        Yrkesinntekter, kapitalinntekter, skattepliktige og skattefrie overføringer, i løpet av kalenderåret.
 classification_reference: |
   ID av en klassifikasjon eller kodeliste fra KLASS som beskriver verdiene variabelen kan anta.
   For eksempel vil variabelen 'Sivilstand' ha klassifikasjon 'Standard for sivilstand' (kan vises på https://www.ssb.no/klass/klassifikasjoner/19 ) som har ID 19.
+  -------------------------
+  >>> EKSEMPEL: "19"
 unit_types: |
-  Enhetstyper - enhetene som beskrives av denne variabelen. Variabelen “sivilstand” vil f.eks. ha enhetstypen person, mens f.eks. “Produsentpris for tjenester” vil ha både foretak og bedrift som enhetstyper siden variabelen kan beskrive begge.
+  Enhetstyper - enhetene som beskrives av denne variabelen. Variabelen “sivilstand” vil f.eks. ha enhetstypen person,
+  mens f.eks. “Produsentpris for tjenester” vil ha både foretak og bedrift som enhetstyper siden variabelen kan beskrive begge.
   Verdier skal være koder fra: https://www.ssb.no/klass/klassifikasjoner/702.
+  -------------------------
+  >>> EKSEMPEL:
+    - "20"
 subject_fields: |
   Statistikkområder som variabelen brukes innenfor. For eksempel tilhører variabelen “Sivilstand” statistikkområdet “Befolkning”.
   Verdier skal være koder fra https://www.ssb.no/klass/klassifikasjoner/618.
+  -------------------------
+  >>> EKSEMPEL:
+    - "bf"
+    - "be"
 contains_special_categories_of_personal_data: |
   Viser om variabelen inneholder spesielt sensitive personopplysninger.
-  Kategorier:
-    - opplysninger om etnisk opprinnelse
-    - opplysninger om politisk oppfatning
-    - opplysninger om religion
-    - opplysninger om filosofisk overbevisning
-    - opplysninger om fagforeningsmedlemskap
-    - genetiske opplysninger
-    - biometriske opplysninger med det formål å entydig identifisere noen
-    - helseopplysninger
-    - opplysninger om seksuelle forhold
-    - opplysninger om seksuell legning
-  ref: https://lovdata.no/dokument/NL/lov/2018-06-15-38/KAPITTEL_gdpr-2#gdpr/a9
+  -------------------------
+  >>> EKSEMPEL: true
 measurement_type: |
   Måletype som en kvantitativ variabelen tilhører, f.eks.  valuta, areal osv.
   Verdien skal være en kode fra: https://www.ssb.no/klass/klassifikasjoner/303
+  -------------------------
+  >>> EKSEMPEL: "03"
 valid_from: |
   Datoen variabeldefinisjonen er gyldig f.o.m.
+  -------------------------
+  >>> EKSEMPEL: 1999-01-30
 valid_until: |
   Datoen variabeldefinisjonens var gyldig t.o.m. Settes hvis definisjonen skal erstattet av en ny definisjon (med en ny gyldighetsperiode), eller variabelen ikke lenger skal brukes.
+  -------------------------
+  >>> EKSEMPEL: 2024-10-23
 external_reference_uri:  |
   En peker (URI) til ekstern definisjon/dokumentasjon, f.eks. ei webside som er relevant for variabelen.
+  -----------------------------------------------------
+  >>> EKSEMPEL: "https://www.landbruksdirektoratet.com"
 comment: |
-  Her kan en sette inn eventuelle tilleggsopplysninger som ikke hører hjemme i selve definisjonen. Variabelen “Landbakgrunn” har f.eks. merknaden “Fra og med 1.1.2003 ble definisjon endret til også å trekke inn besteforeldrenes fødeland”.
+  Her kan en sette inn eventuelle tilleggsopplysninger som ikke hører hjemme i selve definisjonen.
+  Variabelen “Landbakgrunn” har f.eks. merknaden “Fra og med 1.1.2003 ble definisjon endret til også å trekke inn besteforeldrenes fødeland”.
+  -----------------------------------------------------------------------------------------------
+  >>> EKSEMPEL:
+  comment:
+    nb: |-
+        Fra og med 1.1.2003 ble definisjon endret til også å trekke inn besteforeldrenes fødeland.
 related_variable_definition_uris: |
-  Her kan en legge inn URIer til andre variabler som er relevante. Eksempelvis er variabelen “Inntekt etter skatt” en beregnet variabel der  “Yrkesinntekter” og “Kapitalinntekter” inngår i beregningen. En kan da legge inn deres URI-er i dette feltet.
+  Her kan en legge inn URIer til andre variabler som er relevante. Eksempelvis er variabelen “Inntekt etter skatt” en beregnet variabel der  “Yrkesinntekter” og “Kapitalinntekter” inngår i beregningen.
+  En kan da legge inn deres URI-er i dette feltet.
+  -------------------------
+  >>> EKSEMPEL:
+    - "https://example.com/"
 contact: |
   Her dokumenterer en navn og epost for person eller gruppe som kan svare på spørsmål.
+  -------------------------
+  >>> EKSEMPEL:
+  contact:
+    title:
+      nb: |-
+          Seksjonsleder
+    email: leder@ssb.no
 variable_status: |
   Livssyklus for variabelen.
 id: |

dapla_metadata/variable_definitions/vardef.py

@@ -108,7 +108,7 @@ class Vardef:
         )
 
         logger.info(
-            "Successfully created variable definition '%s' with ID '%s'",
+            "✅ Successfully created variable definition '%s' with ID '%s'",
             new_variable.short_name,
             new_variable.id,
         )
@@ -165,7 +165,7 @@ class Vardef:
         )
 
         logger.info(
-            "Successfully migrated variable definition '%s' with ID '%s'",
+            "✅ Successfully migrated variable definition '%s' with ID '%s'",
             migrated_variable.short_name,
             migrated_variable.id,
         )
@@ -273,7 +273,7 @@ class Vardef:
             custom_directory=Path(custom_file_path) if custom_file_path else None,
         )
         logger.info(
-            f"Created editable variable definition template file at {file_path}",  # noqa: G004
+            f"✅ Created editable variable definition template file at {file_path}",  # noqa: G004
         )
         return file_path
 

dapla_metadata/variable_definitions/variable_definition.py

@@ -34,6 +34,8 @@ from dapla_metadata.variable_definitions._generated.vardef_client.models.variabl
 )
 from dapla_metadata.variable_definitions._utils import config
 from dapla_metadata.variable_definitions._utils._client import VardefClient
+from dapla_metadata.variable_definitions._utils.files import configure_yaml
+from dapla_metadata.variable_definitions._utils.files import pre_process_data
 from dapla_metadata.variable_definitions._utils.variable_definition_files import (
     _read_file_to_model,
 )
@@ -126,7 +128,7 @@ class VariableDefinition(CompleteResponse):
         self.__dict__.update(updated)
 
         logger.info(
-            "Successfully updated variable definition '%s' with ID '%s'",
+            "✅ Successfully updated variable definition '%s' with ID '%s'",
             updated.short_name,
             updated.id,
         )
@@ -176,7 +178,7 @@ class VariableDefinition(CompleteResponse):
             variable_definition_id=self.id,
             active_group=config.get_active_group(),
         )
-        return f"Variable {self.id} safely deleted"
+        return f"✅ Variable {self.id} safely deleted"
 
     @vardef_exception_handler
     def get_patch(self, patch_id: int) -> "VariableDefinition":
@@ -205,9 +207,9 @@ class VariableDefinition(CompleteResponse):
 
         Patches are to be used for minor changes which don't require a new Validity Period.
         Examples of reasons for creating a new Patch:
-          - Correcting a typo
-          - Adding a translation
-          - Adding a subject field
+        - Correcting a typo
+        - Adding a translation
+        - Adding a subject field
 
         Supply only the fields to be changed. Other fields will retain their current values.
 
@@ -234,7 +236,7 @@ class VariableDefinition(CompleteResponse):
         self.__dict__.update(new_patch)
 
         logger.info(
-            "Successfully created patch with patch ID '%s' for variable definition '%s' with ID '%s'",
+            "✅ Successfully created patch with patch ID '%s' for variable definition '%s' with ID '%s'",
             new_patch.patch_id,
             new_patch.short_name,
             new_patch.id,
@@ -309,7 +311,7 @@ class VariableDefinition(CompleteResponse):
         self.__dict__.update(new_validity_period)
 
         logger.info(
-            "Successfully created validity period that is valid from '%s' for variable definition '%s' with ID '%s'",
+            "✅ Successfully created validity period that is valid from '%s' for variable definition '%s' with ID '%s'",
             new_validity_period.valid_from,
             new_validity_period.short_name,
             new_validity_period.id,
@@ -350,7 +352,7 @@ class VariableDefinition(CompleteResponse):
             UpdateDraft(variable_status=VariableStatus.PUBLISHED_INTERNAL),
         )
         logger.info(
-            "Variable definition '%s' with ID '%s' successfully published, new status: %s",
+            "✅ Variable definition '%s' with ID '%s' successfully published, new status: %s",
             update.short_name,
             update.id,
             update.variable_status,
@@ -373,7 +375,7 @@ class VariableDefinition(CompleteResponse):
                 Patch(variable_status=VariableStatus.PUBLISHED_EXTERNAL),
             )
         logger.info(
-            "Variable definition '%s' with ID '%s' successfully published, new status: %s",
+            "✅ Variable definition '%s' with ID '%s' successfully published, new status: %s",
             update.short_name,
             update.id,
             update.variable_status,
@@ -387,7 +389,7 @@ class VariableDefinition(CompleteResponse):
         )
         self.set_file_path(file_path)
         logger.info(
-            f"Created editable variable definition file at {file_path}",  # noqa: G004
+            f"✅ Created editable variable definition file at {file_path}",  # noqa: G004
         )
         return self
 
@@ -405,16 +407,12 @@ class VariableDefinition(CompleteResponse):
 
     def _convert_to_yaml_output(self) -> str:
         stream = StringIO()
-        with ruamel.yaml.YAML(
-            output=stream,
-        ) as yaml:
-            yaml.default_flow_style = False
-            yaml.allow_unicode = True
-            yaml.dump(
-                self.model_dump(
-                    mode="json",
-                    serialize_as_any=True,
-                    warnings="error",
-                ),
+        with ruamel.yaml.YAML(output=stream) as yaml:
+            configure_yaml(yaml)
+            data = self.model_dump(
+                mode="json",
+                serialize_as_any=True,
+                warnings="error",
             )
+            yaml.dump(pre_process_data(data))
         return stream.getvalue()

{dapla_toolbelt_metadata-0.6.2.dist-info → dapla_toolbelt_metadata-0.6.4.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.3
 Name: dapla-toolbelt-metadata
-Version: 0.6.2
+Version: 0.6.4
 Summary: Dapla Toolbelt Metadata
 License: MIT
 Author: Team Metadata
 Author-email: metadata@ssb.no
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.10
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -24,7 +24,7 @@ Requires-Dist: pyjwt (>=2.8.0)
 Requires-Dist: python-dotenv (>=1.0.1)
 Requires-Dist: requests (>=2.31.0)
 Requires-Dist: ruamel-yaml (>=0.18.10)
-Requires-Dist: ssb-datadoc-model (>=6.0.0,<7.0.0)
+Requires-Dist: ssb-datadoc-model (==6.0.0)
 Requires-Dist: ssb-klass-python (>=1.0.1)
 Requires-Dist: typing-extensions (>=4.12.2)
 Project-URL: Changelog, https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases

{dapla_toolbelt_metadata-0.6.2.dist-info → dapla_toolbelt_metadata-0.6.4.dist-info}/RECORD

@@ -7,7 +7,7 @@ dapla_metadata/dapla/__init__.py,sha256=tkapF-YwmruPPrKvN3pEoCZqb7xvJx_ogBM8XyGM
 dapla_metadata/dapla/user_info.py,sha256=bENez-ICt9ySR8orYebO68Q3_2LkIW9QTL58DTctmEQ,4833
 dapla_metadata/datasets/__init__.py,sha256=TvzskpdFC6hGcC9_55URT5jr5wNAPzXuISd2UjJWM_8,280
 dapla_metadata/datasets/code_list.py,sha256=kp1O6sUiUAP9WKlWY8IgHWx_1IOzJA63WveHqolgKmg,9082
-dapla_metadata/datasets/core.py,sha256=gp-zTJyn5Z_9bbmHF749OUx0kpYviiDAP9MDwSZiRW8,21768
+dapla_metadata/datasets/core.py,sha256=WfBIkNWxXH_WdCyZCsO3o-CRjb2LbsMJp-P01-b8nYw,25706
 dapla_metadata/datasets/dapla_dataset_path_info.py,sha256=zdkVjxlqXMBe7eTAneUrTDP0_fx7JsEQ_0JrKjREhfU,26854
 dapla_metadata/datasets/dataset_parser.py,sha256=bc3KOIDQGgdZMPh3XVHhiKMsY6FxIY9glvGlwTM4g7I,8233
 dapla_metadata/datasets/external_sources/__init__.py,sha256=qvIdXwqyEmXNUCB94ZtZXRzifdW4hiXASFFPtC70f6E,83
@@ -21,10 +21,10 @@ dapla_metadata/datasets/utility/constants.py,sha256=SqZMc1v8rO2b_nRFJR7frVd0TAGv
 dapla_metadata/datasets/utility/enums.py,sha256=SpV4xlmP1YMaJPbmX03hqRLHUOhXIk5gquTeJ8G_5OE,432
 dapla_metadata/datasets/utility/utils.py,sha256=fAevz9X0PHw-JL0_4V0geTkoBV31qoO7-dVLFJaIfbo,18370
 dapla_metadata/standards/__init__.py,sha256=n8jnMrudLuScSdfQ4UMJorc-Ptg3Y1-ilT8zAaQnM70,179
-dapla_metadata/standards/name_validator.py,sha256=VercKsO7bNDEfAXy_oxMadL8KLVhxmmTe0T8sNlwzm0,9179
+dapla_metadata/standards/name_validator.py,sha256=6-DQE_EKVd6UjL--EXpFcZDQtusVbSFaWaUY-CfOV2c,9184
 dapla_metadata/standards/standard_validators.py,sha256=tcCiCI76wUVtMzXA2oCgdauZc0uGgUi11FKu-t7KGwQ,3767
 dapla_metadata/standards/utils/__init__.py,sha256=AiM7JcpFsAgyuCyLDYZo9kI94wvIImMDGoV2lKhS4pE,42
-dapla_metadata/standards/utils/constants.py,sha256=SiWViQo90PXruWeACuW9Y1iNqmFoDVe0dP4WfO29PXU,2498
+dapla_metadata/standards/utils/constants.py,sha256=mhWNFnS6NMsRl0c_deIdzY7_bD_wKn_oej6rzDjgwq4,2578
 dapla_metadata/variable_definitions/__init__.py,sha256=j_Nn5mnlZ2uio9moDFLE2xpALqrYpupIZMlvwbLuEuA,391
 dapla_metadata/variable_definitions/_generated/.openapi-generator/FILES,sha256=hfNllHEkFODP0XbgqZB5Tz2mmEBFeAeMplXXslczo1E,634
 dapla_metadata/variable_definitions/_generated/.openapi-generator/VERSION,sha256=Y6lrqS2bXoujk5K-DCAwRFdRmkCKuTgvlngEx6FY5So,7
@@ -73,17 +73,17 @@ dapla_metadata/variable_definitions/_generated/vardef_client/py.typed,sha256=47D
 dapla_metadata/variable_definitions/_generated/vardef_client/rest.py,sha256=x4PWmg3IYQBr8OgnrWr3l4Ke2rElHP3zAEVxk2U-mOc,12022
 dapla_metadata/variable_definitions/_utils/__init__.py,sha256=qAhRLJoTBqtR3f9xRXTRhD7-5Xg0Opk1Ks5F4AUYnpA,45
 dapla_metadata/variable_definitions/_utils/_client.py,sha256=v1-9VjrdPI6-sroam5vXMPEV1dQMPsYk7KyGd48HjYw,971
-dapla_metadata/variable_definitions/_utils/config.py,sha256=PZIe0XsrYKsF5SiATCx5EoSn-s9EbYAteiA2k5FxRTw,2475
-dapla_metadata/variable_definitions/_utils/constants.py,sha256=b0HOSPeTbY0Zjs10YIOMDx8MsSQF12ES9tvXlj8VTao,1483
+dapla_metadata/variable_definitions/_utils/config.py,sha256=BpLrnuqgtqz_kxBc_Kd-I1QNL7y2RxRXgX-IVbMIclQ,2416
+dapla_metadata/variable_definitions/_utils/constants.py,sha256=BGITkRNYtRDySM-anDMQDvO2JrXm3lDjw7ZmYfhFlXU,1884
 dapla_metadata/variable_definitions/_utils/descriptions.py,sha256=bB5QHNc4eOhmpLQHCty-CP5_aA82chkICifXw430suI,2746
-dapla_metadata/variable_definitions/_utils/files.py,sha256=sK6rYKJLGp3nSySLPM9WX5SFOFSit_envi9knUmZWi0,9253
-dapla_metadata/variable_definitions/_utils/template_files.py,sha256=hCINDyGFYJtFaWP-0Qjbrv0S6GFkPccIDyKD0KcPtuA,3670
-dapla_metadata/variable_definitions/_utils/variable_definition_files.py,sha256=ePlbsrVl1JNMDUomS-ldYOeOilmcjQy0I5RhorShE2o,2785
+dapla_metadata/variable_definitions/_utils/files.py,sha256=qdO9D0l-6FnSGZImTtyMsrFfauFqvQyCWz0knLSklbo,13193
+dapla_metadata/variable_definitions/_utils/template_files.py,sha256=-PgYs4TG4vrXLQgk47pow9ZsqlZqhtO755LnEmvN4MA,3405
+dapla_metadata/variable_definitions/_utils/variable_definition_files.py,sha256=PbqsFdHxsq0EWBg9s2Y57LqVP7aPmGD5-FZfnzuOw2Q,4078
 dapla_metadata/variable_definitions/exceptions.py,sha256=z6Gtd84FboDu7vWjC3wathIF7I0gF0imtRhwMkr16lY,7851
-dapla_metadata/variable_definitions/resources/vardef_model_descriptions_nb.yaml,sha256=R5X27c32_Y9WWR1xa-WTYlaVhj9b-p6_gzF_eEWFG_U,4557
-dapla_metadata/variable_definitions/vardef.py,sha256=ZBV8ezGRJ8rsmb2H-ZyVX3-DNnMDPIZsHa7MzlcEkNM,11340
-dapla_metadata/variable_definitions/variable_definition.py,sha256=Q_XjMKaJhPwoM0vyfgYVWKYNAYFX0caNWWthXmQxeNs,14784
-dapla_toolbelt_metadata-0.6.2.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
-dapla_toolbelt_metadata-0.6.2.dist-info/METADATA,sha256=k9T7kJSEktMYpwiz0ZRO8Ehq1ZE6PimoDOYu4hTOuXI,4917
-dapla_toolbelt_metadata-0.6.2.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-dapla_toolbelt_metadata-0.6.2.dist-info/RECORD,,
+dapla_metadata/variable_definitions/resources/vardef_model_descriptions_nb.yaml,sha256=z-P9q0yVk8mcKIMPByEhdF3q-OQzd26jPgrpLep4cU0,5223
+dapla_metadata/variable_definitions/vardef.py,sha256=KYd31nCGhxuzC0hpKR6foQjO39Tlb3vu9IDqUoMvTeY,11352
+dapla_metadata/variable_definitions/variable_definition.py,sha256=sj49uot0e4UJW4QJ3dEJGgjY4yfCHOkxS2NdD2t60b8,14883
+dapla_toolbelt_metadata-0.6.4.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
+dapla_toolbelt_metadata-0.6.4.dist-info/METADATA,sha256=Bq3nNW2h1USHM-xn1eLag-kHCqYn6l7PtwZqqEE4_bk,4905
+dapla_toolbelt_metadata-0.6.4.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+dapla_toolbelt_metadata-0.6.4.dist-info/RECORD,,