dapla-toolbelt-metadata 0.6.2__tar.gz → 0.6.4__tar.gz

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.4}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.3
 Name: dapla-toolbelt-metadata
-Version: 0.6.2
+Version: 0.6.4
 Summary: Dapla Toolbelt Metadata
 License: MIT
 Author: Team Metadata
 Author-email: metadata@ssb.no
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.10
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -24,7 +24,7 @@ Requires-Dist: pyjwt (>=2.8.0)
 Requires-Dist: python-dotenv (>=1.0.1)
 Requires-Dist: requests (>=2.31.0)
 Requires-Dist: ruamel-yaml (>=0.18.10)
-Requires-Dist: ssb-datadoc-model (>=6.0.0,<7.0.0)
+Requires-Dist: ssb-datadoc-model (==6.0.0)
 Requires-Dist: ssb-klass-python (>=1.0.1)
 Requires-Dist: typing-extensions (>=4.12.2)
 Project-URL: Changelog, https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.4}/pyproject.toml

@@ -1,36 +1,45 @@
-[tool.poetry]
+[project]
 name = "dapla-toolbelt-metadata"
-version = "0.6.2"
 description = "Dapla Toolbelt Metadata"
-authors = ["Team Metadata <metadata@ssb.no>"]
 license = "MIT"
+version = "0.6.4"
+dynamic = ["classifiers"]
 readme = "README.md"
+authors = [{ name = "Team Metadata", email = "metadata@ssb.no" }]
+requires-python = '>=3.10'
+dependencies = [
+    'pyarrow >=8.0.0',
+    'pydantic >=2.5.2',
+    'arrow >=1.3.0',
+    'python-dotenv >=1.0.1',
+    'requests >=2.31.0',
+    'beautifulsoup4 >=4.12.3',
+    'cloudpathlib[gs] >=0.17.0',
+    'pyjwt >=2.8.0',
+    'ssb-klass-python >=1.0.1',
+    'ssb-datadoc-model ==6.0.0',
+    'typing-extensions >=4.12.2',
+    'ruamel-yaml >=0.18.10',
+    'google-auth >=2.38.0',
+    'lxml >=5.3.1',
+]
+
+[project.urls]
 homepage = "https://github.com/statisticsnorway/dapla-toolbelt-metadata"
 repository = "https://github.com/statisticsnorway/dapla-toolbelt-metadata"
 documentation = "https://statisticsnorway.github.io/dapla-toolbelt-metadata"
-classifiers = ["Development Status :: 4 - Beta"]
+Changelog = "https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases"
 
+[tool.poetry]
+requires-poetry = '>=2.0,<3.0'
 packages = [{ include = "dapla_metadata", from = "src" }]
+classifiers = ["Development Status :: 4 - Beta"]
 
-[tool.poetry.urls]
-Changelog = "https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases"
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.9.0" # Used with nox
 
 [tool.poetry.dependencies]
-python = ">=3.10,<4.0"
-pyarrow = ">=8.0.0"
-pydantic = ">=2.5.2"
-arrow = ">=1.3.0"
-python-dotenv = ">=1.0.1"
-requests = ">=2.31.0"
-beautifulsoup4 = ">=4.12.3"
-cloudpathlib = { extras = ["gs"], version = ">=0.17.0" }
-pyjwt = ">=2.8.0"
-ssb-klass-python = ">=1.0.1"
-ssb-datadoc-model = "^6.0.0"
-typing-extensions = ">=4.12.2"
-ruamel-yaml = ">=0.18.10"
-google-auth = ">=2.38.0"
-lxml = ">=5.3.1"
+python = ">=3.10,<4.0" # This is currently needed for compatibility with ssb-klass-python
 
 [tool.poetry.group.dev.dependencies]
 pygments = ">=2.18.0"
@@ -64,7 +73,7 @@ types-beautifulsoup4 = ">=4.12.0.20240511"
 ipykernel = "^6.29.5"
 rich = "^13.9.4"
 bpython = "^0.24"
-testcontainers = { version = "^4.8.2", extras = ["generic"] }
+testcontainers = { version = ">=4.8.2", extras = ["generic"] }
 pytest-asyncio = "^0.26.0"
 
 [tool.pytest.ini_options]
@@ -77,11 +86,7 @@ tests = ["tests", "*/tests", "*/test"]
 [tool.coverage.run]
 branch = true
 source = ["dapla_metadata"]
-omit = [
-    "tests/*",
-    "__init__.py",
-    "*/dapla_metadata/variable_definitions/_generated/*",
-]
+omit = ["tests/*", "__init__.py", "*/dapla_metadata/variable_definitions/_generated/*"]
 relative_files = true
 
 [tool.coverage.report]
@@ -105,7 +110,6 @@ warn_unreachable = false
 ignore_errors = true
 disable_error_code = ["unreachable"]
 
-
 [[tool.mypy.overrides]]
 # Allow missing type hints in third-party libraries without type information.
 module = [
@@ -127,27 +131,15 @@ ignore_missing_imports = true
 # Also don't require type annotations
 [[tool.mypy.overrides]]
 module = ["tests.*"]
-disable_error_code = [
-    "var-annotated",
-    "has-type",
-    "no-any-return",
-    "no-untyped-def",
-]
+disable_error_code = ["var-annotated", "has-type", "no-any-return", "no-untyped-def"]
 
 [tool.ruff]
-force-exclude = true # Apply excludes to pre-commit
+force-exclude = true                                                                                       # Apply excludes to pre-commit
 show-fixes = true
 src = ["src", "tests"]
-target-version = "py310" # Minimum Python version supported
+target-version = "py310"                                                                                   # Minimum Python version supported
 include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"]
-extend-exclude = [
-    "__pycache__",
-    "old",
-    ".ipynb_checkpoints",
-    "noxfile.py",
-    "docs/conf.py",
-    "_generated/",
-]
+extend-exclude = ["__pycache__", "old", ".ipynb_checkpoints", "noxfile.py", "docs/conf.py", "_generated/"]
 
 # Ruff rules may be customized as desired: https://docs.astral.sh/ruff/rules/
 [tool.ruff.lint]
@@ -181,12 +173,7 @@ fixture-parentheses = false
 max-args = 8
 
 [tool.ruff.lint.pep8-naming]
-classmethod-decorators = [
-    "classmethod",
-    "validator",
-    "root_validator",
-    "pydantic.validator",
-]
+classmethod-decorators = ["classmethod", "validator", "root_validator", "pydantic.validator"]
 
 [tool.ruff.lint.per-file-ignores]
 "*/__init__.py" = ["F401"]
@@ -206,5 +193,5 @@ classmethod-decorators = [
 ]
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core>=2.0"]
 build-backend = "poetry.core.masonry.api"

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.4}/src/dapla_metadata/datasets/core.py

@@ -109,8 +109,11 @@ class Datadoc:
         self.dataset_path: pathlib.Path | CloudPath | None = None
         self.dataset = model.Dataset()
         self.variables: list = []
+        self.pseudo_variables: list[model.PseudoVariable] = []
         self.variables_lookup: dict[str, model.Variable] = {}
+        self.pseudo_variables_lookup: dict[str, model.PseudoVariable] = {}
         self.explicitly_defined_metadata_document = False
+        self.dataset_consistency_status: list = []
         if metadata_document_path:
             self.metadata_document = normalize_path(metadata_document_path)
             self.explicitly_defined_metadata_document = True
@@ -148,11 +151,19 @@ class Datadoc:
         """
         extracted_metadata: model.DatadocMetadata | None = None
         existing_metadata: model.DatadocMetadata | None = None
+        existing_pseudonymization: model.PseudonymizationMetadata | None = None
+
         if self.metadata_document and self.metadata_document.exists():
             existing_metadata = self._extract_metadata_from_existing_document(
                 self.metadata_document,
             )
 
+            existing_pseudonymization = (
+                self._extract_pseudonymization_from_existing_document(
+                    self.metadata_document,
+                )
+            )
+
         if (
             self.dataset_path is not None
             and self.dataset == model.Dataset()
@@ -169,11 +180,14 @@ class Datadoc:
             and existing_metadata is not None
         ):
             existing_file_path = self._get_existing_file_path(extracted_metadata)
-            self._check_ready_to_merge(
+            self.dataset_consistency_status = self._check_dataset_consistency(
                 self.dataset_path,
                 Path(existing_file_path),
                 extracted_metadata,
                 existing_metadata,
+            )
+            self._check_ready_to_merge(
+                self.dataset_consistency_status,
                 errors_as_warnings=self.errors_as_warnings,
             )
             merged_metadata = self._merge_metadata(
@@ -188,10 +202,15 @@ class Datadoc:
             self._set_metadata(merged_metadata)
         else:
             self._set_metadata(existing_metadata or extracted_metadata)
+
+        if existing_pseudonymization:
+            self._set_pseudonymization_metadata(existing_pseudonymization)
+
         set_default_values_variables(self.variables)
         set_default_values_dataset(self.dataset)
         set_dataset_owner(self.dataset)
         self._create_variables_lookup()
+        self._create_pseudo_variables_lookup()
 
     def _get_existing_file_path(
         self,
@@ -218,35 +237,48 @@ class Datadoc:
         self.dataset = merged_metadata.dataset
         self.variables = merged_metadata.variables
 
+    def _set_pseudonymization_metadata(
+        self,
+        existing_pseudonymization: model.PseudonymizationMetadata | None,
+    ) -> None:
+        if not existing_pseudonymization or not (
+            existing_pseudonymization.pseudo_variables
+        ):
+            msg = "Could not read pseudonymization metadata"
+            raise ValueError(msg)
+        self.pseudo_variables = existing_pseudonymization.pseudo_variables
+
     def _create_variables_lookup(self) -> None:
         self.variables_lookup = {
             v.short_name: v for v in self.variables if v.short_name
         }
 
+    def _create_pseudo_variables_lookup(self) -> None:
+        self.pseudo_variables_lookup = {
+            v.short_name: v for v in self.pseudo_variables if v.short_name
+        }
+
     @staticmethod
-    def _check_ready_to_merge(
+    def _check_dataset_consistency(
         new_dataset_path: Path | CloudPath,
         existing_dataset_path: Path,
         extracted_metadata: model.DatadocMetadata,
         existing_metadata: model.DatadocMetadata,
-        *,
-        errors_as_warnings: bool,
-    ) -> None:
-        """Check if the datasets are consistent enough to make a successful merge of metadata.
+    ) -> list[dict[str, object]]:
+        """Run consistency tests.
 
         Args:
             new_dataset_path: Path to the dataset to be documented.
             existing_dataset_path: Path stored in the existing metadata.
             extracted_metadata: Metadata extracted from a physical dataset.
             existing_metadata: Metadata from a previously created metadata document.
-            errors_as_warnings: True if failing checks should be raised as warnings, not errors.
 
-        Raises:
-            InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+        Returns:
+            List if dict with property name and boolean success flag
         """
         new_dataset_path_info = DaplaDatasetPathInfo(new_dataset_path)
         existing_dataset_path_info = DaplaDatasetPathInfo(existing_dataset_path)
-        results = [
+        return [
             {
                 "name": "Bucket name",
                 "success": (
@@ -290,6 +322,20 @@ class Datadoc:
                 ),
             },
         ]
+
+    @staticmethod
+    def _check_ready_to_merge(
+        results: list[dict[str, object]], *, errors_as_warnings: bool
+    ) -> None:
+        """Check if the datasets are consistent enough to make a successful merge of metadata.
+
+        Args:
+            results: List if dict with property name and boolean success flag
+            errors_as_warnings: True if failing checks should be raised as warnings, not errors.
+
+        Raises:
+            InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+        """
         if failures := [result for result in results if not result["success"]]:
             msg = f"{INCONSISTENCIES_MESSAGE} {', '.join(str(f['name']) for f in failures)}"
             if errors_as_warnings:
@@ -384,6 +430,42 @@ class Datadoc:
             )
             return None
 
+    def _extract_pseudonymization_from_existing_document(
+        self,
+        document: pathlib.Path | CloudPath,
+    ) -> model.PseudonymizationMetadata | None:
+        """Read pseudo metadata from an existing metadata document.
+
+        If there is pseudo metadata in the document supplied, the method validates and returns the pseudonymization structure.
+
+        Args:
+            document: A path to the existing metadata document.
+
+        Raises:
+            json.JSONDecodeError: If the metadata document cannot be parsed.
+        """
+        try:
+            with document.open(mode="r", encoding="utf-8") as file:
+                fresh_metadata = json.load(file)
+        except json.JSONDecodeError:
+            logger.warning(
+                "Could not open existing metadata file %s.",
+                document,
+                exc_info=True,
+            )
+            return None
+
+        if not is_metadata_in_container_structure(fresh_metadata):
+            return None
+
+        pseudonymization_metadata = fresh_metadata.get("pseudonymization")
+        if pseudonymization_metadata is None:
+            return None
+
+        return model.PseudonymizationMetadata.model_validate_json(
+            json.dumps(pseudonymization_metadata),
+        )
+
     def _extract_subject_field_from_path(
         self,
         dapla_dataset_path_info: DaplaDatasetPathInfo,
@@ -501,6 +583,11 @@ class Datadoc:
         )
         if self.container:
             self.container.datadoc = datadoc
+            if not self.container.pseudonymization:
+                self.container.pseudonymization = model.PseudonymizationMetadata(
+                    pseudo_dataset=model.PseudoDataset()
+                )
+            self.container.pseudonymization.pseudo_variables = self.pseudo_variables
         else:
             self.container = model.MetadataContainer(datadoc=datadoc)
         if self.metadata_document:
@@ -530,3 +617,16 @@ class Datadoc:
             self.dataset,
         ) + num_obligatory_variables_fields_completed(self.variables)
         return calculate_percentage(num_set_fields, num_all_fields)
+
+    def add_pseudo_variable(self, variable_short_name: str) -> None:
+        """Adds a new pseudo variable to the list of pseudonymized variables."""
+        if self.variables_lookup[variable_short_name] is not None:
+            pseudo_variable = model.PseudoVariable(short_name=variable_short_name)
+            self.pseudo_variables.append(pseudo_variable)
+            self.pseudo_variables_lookup[variable_short_name] = pseudo_variable
+
+    def get_pseudo_variable(
+        self, variable_short_name: str
+    ) -> model.PseudoVariable | None:
+        """Finds a pseudo variable by shortname."""
+        return self.pseudo_variables_lookup.get(variable_short_name)

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.4}/src/dapla_metadata/standards/name_validator.py

@@ -128,9 +128,9 @@ class NamingStandardReport:
         """Returns an appropriate message based on the success rate."""
         rate = self.success_rate()
         if rate is not None:
-            if rate == 100:
+            if 95 <= rate <= 100:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_BEST
-            if 70 < rate < 100:
+            if 70 < rate < 95:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_GOOD
             if 40 <= rate <= 70:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_AVERAGE

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.4}/src/dapla_metadata/standards/utils/constants.py

@@ -9,7 +9,7 @@ NAME_STANDARD_SUCCESS = "Filene dine er i samsvar med SSB-navnestandarden"
 NAME_STANDARD_VIOLATION = "Det er oppdaget brudd på SSB-navnestandard:"
 
 MISSING_BUCKET_NAME = "Filnavn mangler bøttenavn ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
-MISSING_VERSION = "Filnavn mangler versjon ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
+MISSING_VERSION = "Filnavn mangler versjon, hvis ikke filen er nyeste versjon kan dette være brudd på navnestandarden ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#versjonering-av-datasett"
 MISSING_PERIOD = "Filnavn mangler gyldighetsperiode ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
 MISSING_SHORT_NAME = "Kortnavn for statistikk mangler ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
 MISSING_DATA_STATE = "Mappe for datatilstand mangler ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
@@ -26,7 +26,7 @@ BUCKET_NAME_UNKNOWN = "Kan ikke validere bøttenavn"
 
 SSB_NAMING_STANDARD_REPORT = "SSB navnestandard rapport"
 SSB_NAMING_STANDARD_REPORT_SUCCESS_RATE = "Suksess rate"
-SSB_NAMING_STANDARD_REPORT_RESULT_BEST = "🚀 Fantastisk! Alt bestått! 🎉\n"
+SSB_NAMING_STANDARD_REPORT_RESULT_BEST = "🚀 Fantastisk! 🎉\n"
 SSB_NAMING_STANDARD_REPORT_RESULT_GOOD = (
     "✅ Bra jobba! Fortsatt litt rom for forbedring. 😊\n"
 )

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.4}/src/dapla_metadata/variable_definitions/_utils/config.py

@@ -59,8 +59,7 @@ def get_vardef_host() -> str:
         case DaplaEnvironment.TEST:
             return VARDEF_HOST_TEST
         case DaplaEnvironment.DEV:
-            msg = "Vardef is not available in dev."
-            raise NotImplementedError(msg)
+            return VARDEF_HOST_TEST
         case _:
             return get_config_item("VARDEF_HOST") or "http://localhost:8080"
 

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.4}/src/dapla_metadata/variable_definitions/_utils/constants.py

@@ -39,3 +39,26 @@ MACHINE_GENERATED_FIELDS = [
 
 OPTIONAL_FIELD = "~ Valgfritt felt ~"
 REQUIRED_FIELD = "! Obligatorisk felt !"
+
+YAML_STR_TAG = "tag:yaml.org,2002:str"
+
+BLOCK_FIELDS = [
+    "definition",
+    "name",
+    "contact.title",
+    "comment",
+]
+
+DOUBLE_QUOTE_FIELDS = [
+    "unit_types",
+    "subject_fields",
+    "related_variable_definition_uris",
+    "owner",
+    "short_name",
+    "classification_reference",
+    "measurement_type",
+    "external_reference_uri",
+    "created_by",
+    "id",
+    "last_updated_by",
+]

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.4}/src/dapla_metadata/variable_definitions/_utils/files.py

@@ -4,12 +4,16 @@ import logging
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING
+from typing import Any
 from typing import cast
 
 import pytz
 from pydantic.config import JsonDict
 from ruamel.yaml import YAML
 from ruamel.yaml import CommentedMap
+from ruamel.yaml import RoundTripRepresenter
+from ruamel.yaml.scalarstring import DoubleQuotedScalarString
+from ruamel.yaml.scalarstring import LiteralScalarString
 
 from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
     CompleteResponse,
@@ -18,6 +22,8 @@ from dapla_metadata.variable_definitions._generated.vardef_client.models.variabl
     VariableStatus,
 )
 from dapla_metadata.variable_definitions._utils import config
+from dapla_metadata.variable_definitions._utils.constants import BLOCK_FIELDS
+from dapla_metadata.variable_definitions._utils.constants import DOUBLE_QUOTE_FIELDS
 from dapla_metadata.variable_definitions._utils.constants import (
     MACHINE_GENERATED_FIELDS,
 )
@@ -40,6 +46,7 @@ from dapla_metadata.variable_definitions._utils.constants import (
 from dapla_metadata.variable_definitions._utils.constants import (
     VARIABLE_STATUS_FIELD_NAME,
 )
+from dapla_metadata.variable_definitions._utils.constants import YAML_STR_TAG
 from dapla_metadata.variable_definitions._utils.descriptions import (
     apply_norwegian_descriptions_to_model,
 )
@@ -112,6 +119,15 @@ def _get_variable_definitions_dir():
     return folder_path
 
 
+def _set_field_requirement(field_name: str, field: Any) -> str | None:
+    """Determine the field requirement status."""
+    if field_name not in MACHINE_GENERATED_FIELDS:
+        if field.is_required() or field_name == VARIABLE_STATUS_FIELD_NAME:
+            return REQUIRED_FIELD
+        return OPTIONAL_FIELD
+    return None
+
+
 def _populate_commented_map(
     field_name: str,
     value: str,
@@ -120,16 +136,17 @@ def _populate_commented_map(
 ) -> None:
     """Add data to a CommentedMap."""
     commented_map[field_name] = value
-    field = model_instance.model_fields[field_name]
+    field = type(model_instance).model_fields[field_name]
     description: JsonValue = cast(
         JsonDict,
         field.json_schema_extra,
     )[NORWEGIAN_DESCRIPTIONS]
+    field_requirement: str | None = _set_field_requirement(field_name, field)
     if description is not None:
         new_description = (
-            (REQUIRED_FIELD if field.is_required() else OPTIONAL_FIELD)
-            + "\n"
-            + str(description)
+            ("\n" + field_requirement + "\n" + str(description))
+            if field_requirement
+            else ("\n" + str(description))
         )
         commented_map.yaml_set_comment_before_after_key(
             field_name,
@@ -174,14 +191,22 @@ def _validate_and_create_directory(custom_directory: Path) -> Path:
     return custom_directory
 
 
-def _configure_yaml() -> YAML:
-    yaml = YAML()  # Use ruamel.yaml library
-    yaml.default_flow_style = False  # Ensures pretty YAML formatting
-
+def configure_yaml(yaml: YAML) -> YAML:
+    """Common Yaml config for variable definitions."""
+    yaml.Representer = RoundTripRepresenter  # Preserve the order of keys etc.
+    yaml.default_flow_style = False  # Ensures pretty YAML formatting block style
+    yaml.allow_unicode = True  # Support special characters
+    yaml.preserve_quotes = True
+    yaml.width = 4096  # prevent wrapping lines
+    yaml.indent(
+        mapping=4,
+        sequence=6,
+        offset=4,
+    )  # Ensure indentation for nested keys and lists
     yaml.representer.add_representer(
         VariableStatus,
         lambda dumper, data: dumper.represent_scalar(
-            "tag:yaml.org,2002:str",
+            YAML_STR_TAG,
             data.value,
         ),
     )
@@ -189,6 +214,76 @@ def _configure_yaml() -> YAML:
     return yaml
 
 
+def _safe_get(data: dict, keys: list):
+    """Safely navigate nested dictionaries."""
+    for key in keys:
+        if not isinstance(data, dict) or key not in data or data[key] is None:
+            return None
+        data = data[key]
+    return data
+
+
+def _apply_literal_scalars(field: dict):
+    """Helper function to wrap `LanguageStringType` values in `LiteralScalarString`.
+
+    This function wraps each non-`None` language value in a `LanguageStringType` field
+    in the `LiteralScalarString` YAML type, ensuring proper YAML formatting with block style.
+    """
+    for lang, value in field.items():
+        if value is not None:
+            field[lang] = LiteralScalarString(value)
+
+
+def _apply_double_quotes_to_dict_values(field: dict):
+    """Helper function to wrap dictionary values in `DoubleQuotedScalarString`.
+
+    This function wraps each non-`None` value in a dictionary, including values inside lists,
+    in the `DoubleQuotedScalarString` YAML type, ensuring proper YAML formatting with double quotes.
+    """
+    for sub_key, sub_value in field.items():
+        if isinstance(sub_value, list):
+            field[sub_key] = [
+                DoubleQuotedScalarString(item) for item in sub_value if item is not None
+            ]
+        elif sub_value is not None:
+            field[sub_key] = DoubleQuotedScalarString(sub_value)
+
+
+def pre_process_data(data: dict) -> dict:
+    """Format variable definition model fields with ruamel YAML scalar string types.
+
+    This method sets the appropriate scalar string type (either `LiteralScalarString` or `DoubleQuotedScalarString`)
+    for fields of the variable definition model, based on predefined lists of fields.
+
+    It processes both nested dictionaries and lists, ensuring each element is formatted with the correct YAML string type.
+
+    Args:
+        data (dict): A dictionary containing the variable definition data.
+
+    Returns:
+        dict: The updated dictionary with model fields formatted as ruamel.yaml scalar string types.
+    """
+    for key in BLOCK_FIELDS:
+        keys = key.split(".")
+        field = _safe_get(data, keys)
+        if isinstance(field, dict):
+            _apply_literal_scalars(field)
+
+    for key in DOUBLE_QUOTE_FIELDS:
+        keys = key.split(".")
+        field = _safe_get(data, keys)
+        if isinstance(field, list):
+            data[key] = [
+                DoubleQuotedScalarString(item) for item in field if item is not None
+            ]
+        elif isinstance(field, str):
+            data[key] = DoubleQuotedScalarString(data[key])
+        elif isinstance(field, dict):
+            _apply_double_quotes_to_dict_values(field)
+
+    return data
+
+
 def _model_to_yaml_with_comments(
     model_instance: CompleteResponse,
     file_name: str,
@@ -209,7 +304,8 @@ def _model_to_yaml_with_comments(
     Returns:
         Path: The file path of the generated YAML file.
     """
-    yaml = _configure_yaml()
+    yaml = YAML()
+    configure_yaml(yaml)
 
     from dapla_metadata.variable_definitions.variable_definition import (
         VariableDefinition,
@@ -223,7 +319,7 @@ def _model_to_yaml_with_comments(
         serialize_as_any=True,
         warnings="error",
     )
-
+    data = pre_process_data(data)
     # One CommentMap for each section in the yaml file
     machine_generated_map = CommentedMap()
     commented_map = CommentedMap()