dapla-toolbelt-metadata 0.6.3__tar.gz → 0.6.5__tar.gz

{dapla_toolbelt_metadata-0.6.3 → dapla_toolbelt_metadata-0.6.5}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.3
 Name: dapla-toolbelt-metadata
-Version: 0.6.3
+Version: 0.6.5
 Summary: Dapla Toolbelt Metadata
 License: MIT
 Author: Team Metadata
 Author-email: metadata@ssb.no
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.10
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
@@ -24,7 +24,7 @@ Requires-Dist: pyjwt (>=2.8.0)
 Requires-Dist: python-dotenv (>=1.0.1)
 Requires-Dist: requests (>=2.31.0)
 Requires-Dist: ruamel-yaml (>=0.18.10)
-Requires-Dist: ssb-datadoc-model (>=6.0.0,<7.0.0)
+Requires-Dist: ssb-datadoc-model (==6.0.0)
 Requires-Dist: ssb-klass-python (>=1.0.1)
 Requires-Dist: typing-extensions (>=4.12.2)
 Project-URL: Changelog, https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases

{dapla_toolbelt_metadata-0.6.3 → dapla_toolbelt_metadata-0.6.5}/pyproject.toml

@@ -1,36 +1,45 @@
-[tool.poetry]
+[project]
 name = "dapla-toolbelt-metadata"
-version = "0.6.3"
 description = "Dapla Toolbelt Metadata"
-authors = ["Team Metadata <metadata@ssb.no>"]
 license = "MIT"
+version = "0.6.5"
+dynamic = ["classifiers"]
 readme = "README.md"
+authors = [{ name = "Team Metadata", email = "metadata@ssb.no" }]
+requires-python = '>=3.10'
+dependencies = [
+    'pyarrow >=8.0.0',
+    'pydantic >=2.5.2',
+    'arrow >=1.3.0',
+    'python-dotenv >=1.0.1',
+    'requests >=2.31.0',
+    'beautifulsoup4 >=4.12.3',
+    'cloudpathlib[gs] >=0.17.0',
+    'pyjwt >=2.8.0',
+    'ssb-klass-python >=1.0.1',
+    'ssb-datadoc-model ==6.0.0',
+    'typing-extensions >=4.12.2',
+    'ruamel-yaml >=0.18.10',
+    'google-auth >=2.38.0',
+    'lxml >=5.3.1',
+]
+
+[project.urls]
 homepage = "https://github.com/statisticsnorway/dapla-toolbelt-metadata"
 repository = "https://github.com/statisticsnorway/dapla-toolbelt-metadata"
 documentation = "https://statisticsnorway.github.io/dapla-toolbelt-metadata"
-classifiers = ["Development Status :: 4 - Beta"]
+Changelog = "https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases"
 
+[tool.poetry]
+requires-poetry = '>=2.0,<3.0'
 packages = [{ include = "dapla_metadata", from = "src" }]
+classifiers = ["Development Status :: 4 - Beta"]
 
-[tool.poetry.urls]
-Changelog = "https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases"
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.9.0" # Used with nox
 
 [tool.poetry.dependencies]
-python = ">=3.10,<4.0"
-pyarrow = ">=8.0.0"
-pydantic = ">=2.5.2"
-arrow = ">=1.3.0"
-python-dotenv = ">=1.0.1"
-requests = ">=2.31.0"
-beautifulsoup4 = ">=4.12.3"
-cloudpathlib = { extras = ["gs"], version = ">=0.17.0" }
-pyjwt = ">=2.8.0"
-ssb-klass-python = ">=1.0.1"
-ssb-datadoc-model = "^6.0.0"
-typing-extensions = ">=4.12.2"
-ruamel-yaml = ">=0.18.10"
-google-auth = ">=2.38.0"
-lxml = ">=5.3.1"
+python = ">=3.10,<4.0" # This is currently needed for compatibility with ssb-klass-python
 
 [tool.poetry.group.dev.dependencies]
 pygments = ">=2.18.0"
@@ -64,7 +73,7 @@ types-beautifulsoup4 = ">=4.12.0.20240511"
 ipykernel = "^6.29.5"
 rich = "^13.9.4"
 bpython = "^0.24"
-testcontainers = { version = "^4.8.2", extras = ["generic"] }
+testcontainers = { version = ">=4.8.2", extras = ["generic"] }
 pytest-asyncio = "^0.26.0"
 
 [tool.pytest.ini_options]
@@ -77,11 +86,7 @@ tests = ["tests", "*/tests", "*/test"]
 [tool.coverage.run]
 branch = true
 source = ["dapla_metadata"]
-omit = [
-    "tests/*",
-    "__init__.py",
-    "*/dapla_metadata/variable_definitions/_generated/*",
-]
+omit = ["tests/*", "__init__.py", "*/dapla_metadata/variable_definitions/_generated/*"]
 relative_files = true
 
 [tool.coverage.report]
@@ -105,7 +110,6 @@ warn_unreachable = false
 ignore_errors = true
 disable_error_code = ["unreachable"]
 
-
 [[tool.mypy.overrides]]
 # Allow missing type hints in third-party libraries without type information.
 module = [
@@ -127,27 +131,15 @@ ignore_missing_imports = true
 # Also don't require type annotations
 [[tool.mypy.overrides]]
 module = ["tests.*"]
-disable_error_code = [
-    "var-annotated",
-    "has-type",
-    "no-any-return",
-    "no-untyped-def",
-]
+disable_error_code = ["var-annotated", "has-type", "no-any-return", "no-untyped-def"]
 
 [tool.ruff]
-force-exclude = true # Apply excludes to pre-commit
+force-exclude = true                                                                                       # Apply excludes to pre-commit
 show-fixes = true
 src = ["src", "tests"]
-target-version = "py310" # Minimum Python version supported
+target-version = "py310"                                                                                   # Minimum Python version supported
 include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"]
-extend-exclude = [
-    "__pycache__",
-    "old",
-    ".ipynb_checkpoints",
-    "noxfile.py",
-    "docs/conf.py",
-    "_generated/",
-]
+extend-exclude = ["__pycache__", "old", ".ipynb_checkpoints", "noxfile.py", "docs/conf.py", "_generated/"]
 
 # Ruff rules may be customized as desired: https://docs.astral.sh/ruff/rules/
 [tool.ruff.lint]
@@ -181,12 +173,7 @@ fixture-parentheses = false
 max-args = 8
 
 [tool.ruff.lint.pep8-naming]
-classmethod-decorators = [
-    "classmethod",
-    "validator",
-    "root_validator",
-    "pydantic.validator",
-]
+classmethod-decorators = ["classmethod", "validator", "root_validator", "pydantic.validator"]
 
 [tool.ruff.lint.per-file-ignores]
 "*/__init__.py" = ["F401"]
@@ -206,5 +193,5 @@ classmethod-decorators = [
 ]
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core>=2.0"]
 build-backend = "poetry.core.masonry.api"

{dapla_toolbelt_metadata-0.6.3 → dapla_toolbelt_metadata-0.6.5}/src/dapla_metadata/datasets/core.py

@@ -109,7 +109,9 @@ class Datadoc:
         self.dataset_path: pathlib.Path | CloudPath | None = None
         self.dataset = model.Dataset()
         self.variables: list = []
+        self.pseudo_variables: list[model.PseudoVariable] = []
         self.variables_lookup: dict[str, model.Variable] = {}
+        self.pseudo_variables_lookup: dict[str, model.PseudoVariable] = {}
         self.explicitly_defined_metadata_document = False
         self.dataset_consistency_status: list = []
         if metadata_document_path:
@@ -149,11 +151,19 @@ class Datadoc:
         """
         extracted_metadata: model.DatadocMetadata | None = None
         existing_metadata: model.DatadocMetadata | None = None
+        existing_pseudonymization: model.PseudonymizationMetadata | None = None
+
         if self.metadata_document and self.metadata_document.exists():
             existing_metadata = self._extract_metadata_from_existing_document(
                 self.metadata_document,
             )
 
+            existing_pseudonymization = (
+                self._extract_pseudonymization_from_existing_document(
+                    self.metadata_document,
+                )
+            )
+
         if (
             self.dataset_path is not None
             and self.dataset == model.Dataset()
@@ -192,10 +202,15 @@ class Datadoc:
             self._set_metadata(merged_metadata)
         else:
             self._set_metadata(existing_metadata or extracted_metadata)
+
+        if existing_pseudonymization:
+            self._set_pseudonymization_metadata(existing_pseudonymization)
+
         set_default_values_variables(self.variables)
         set_default_values_dataset(self.dataset)
         set_dataset_owner(self.dataset)
         self._create_variables_lookup()
+        self._create_pseudo_variables_lookup()
 
     def _get_existing_file_path(
         self,
@@ -222,11 +237,29 @@ class Datadoc:
         self.dataset = merged_metadata.dataset
         self.variables = merged_metadata.variables
 
+    def _set_pseudonymization_metadata(
+        self,
+        existing_pseudonymization: model.PseudonymizationMetadata | None,
+    ) -> None:
+        if not existing_pseudonymization or not (
+            existing_pseudonymization.pseudo_variables is not None
+        ):
+            msg = "Error reading pseudonymization metadata"
+            logger.error(msg)
+            return
+        self.pseudo_variables = existing_pseudonymization.pseudo_variables
+
     def _create_variables_lookup(self) -> None:
         self.variables_lookup = {
             v.short_name: v for v in self.variables if v.short_name
         }
 
+    def _create_pseudo_variables_lookup(self) -> None:
+        if self.pseudo_variables:
+            self.pseudo_variables_lookup = {
+                v.short_name: v for v in self.pseudo_variables if v.short_name
+            }
+
     @staticmethod
     def _check_dataset_consistency(
         new_dataset_path: Path | CloudPath,
@@ -399,6 +432,42 @@ class Datadoc:
             )
             return None
 
+    def _extract_pseudonymization_from_existing_document(
+        self,
+        document: pathlib.Path | CloudPath,
+    ) -> model.PseudonymizationMetadata | None:
+        """Read pseudo metadata from an existing metadata document.
+
+        If there is pseudo metadata in the document supplied, the method validates and returns the pseudonymization structure.
+
+        Args:
+            document: A path to the existing metadata document.
+
+        Raises:
+            json.JSONDecodeError: If the metadata document cannot be parsed.
+        """
+        try:
+            with document.open(mode="r", encoding="utf-8") as file:
+                fresh_metadata = json.load(file)
+        except json.JSONDecodeError:
+            logger.warning(
+                "Could not open existing metadata file %s.",
+                document,
+                exc_info=True,
+            )
+            return None
+
+        if not is_metadata_in_container_structure(fresh_metadata):
+            return None
+
+        pseudonymization_metadata = fresh_metadata.get("pseudonymization")
+        if pseudonymization_metadata is None:
+            return None
+
+        return model.PseudonymizationMetadata.model_validate_json(
+            json.dumps(pseudonymization_metadata),
+        )
+
     def _extract_subject_field_from_path(
         self,
         dapla_dataset_path_info: DaplaDatasetPathInfo,
@@ -516,6 +585,11 @@ class Datadoc:
         )
         if self.container:
             self.container.datadoc = datadoc
+            if not self.container.pseudonymization:
+                self.container.pseudonymization = model.PseudonymizationMetadata(
+                    pseudo_dataset=model.PseudoDataset()
+                )
+            self.container.pseudonymization.pseudo_variables = self.pseudo_variables
         else:
             self.container = model.MetadataContainer(datadoc=datadoc)
         if self.metadata_document:
@@ -545,3 +619,16 @@ class Datadoc:
             self.dataset,
         ) + num_obligatory_variables_fields_completed(self.variables)
         return calculate_percentage(num_set_fields, num_all_fields)
+
+    def add_pseudo_variable(self, variable_short_name: str) -> None:
+        """Adds a new pseudo variable to the list of pseudonymized variables."""
+        if self.variables_lookup[variable_short_name] is not None:
+            pseudo_variable = model.PseudoVariable(short_name=variable_short_name)
+            self.pseudo_variables.append(pseudo_variable)
+            self.pseudo_variables_lookup[variable_short_name] = pseudo_variable
+
+    def get_pseudo_variable(
+        self, variable_short_name: str
+    ) -> model.PseudoVariable | None:
+        """Finds a pseudo variable by shortname."""
+        return self.pseudo_variables_lookup.get(variable_short_name)

{dapla_toolbelt_metadata-0.6.3 → dapla_toolbelt_metadata-0.6.5}/src/dapla_metadata/variable_definitions/_utils/constants.py

@@ -41,3 +41,24 @@ OPTIONAL_FIELD = "~ Valgfritt felt ~"
 REQUIRED_FIELD = "! Obligatorisk felt !"
 
 YAML_STR_TAG = "tag:yaml.org,2002:str"
+
+BLOCK_FIELDS = [
+    "definition",
+    "name",
+    "contact.title",
+    "comment",
+]
+
+DOUBLE_QUOTE_FIELDS = [
+    "unit_types",
+    "subject_fields",
+    "related_variable_definition_uris",
+    "owner",
+    "short_name",
+    "classification_reference",
+    "measurement_type",
+    "external_reference_uri",
+    "created_by",
+    "id",
+    "last_updated_by",
+]

{dapla_toolbelt_metadata-0.6.3 → dapla_toolbelt_metadata-0.6.5}/src/dapla_metadata/variable_definitions/_utils/files.py

@@ -4,6 +4,7 @@ import logging
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING
+from typing import Any
 from typing import cast
 
 import pytz
@@ -12,7 +13,7 @@ from ruamel.yaml import YAML
 from ruamel.yaml import CommentedMap
 from ruamel.yaml import RoundTripRepresenter
 from ruamel.yaml.scalarstring import DoubleQuotedScalarString
-from ruamel.yaml.scalarstring import FoldedScalarString
+from ruamel.yaml.scalarstring import LiteralScalarString
 
 from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
     CompleteResponse,
@@ -21,6 +22,8 @@ from dapla_metadata.variable_definitions._generated.vardef_client.models.variabl
     VariableStatus,
 )
 from dapla_metadata.variable_definitions._utils import config
+from dapla_metadata.variable_definitions._utils.constants import BLOCK_FIELDS
+from dapla_metadata.variable_definitions._utils.constants import DOUBLE_QUOTE_FIELDS
 from dapla_metadata.variable_definitions._utils.constants import (
     MACHINE_GENERATED_FIELDS,
 )
@@ -116,6 +119,15 @@ def _get_variable_definitions_dir():
     return folder_path
 
 
+def _set_field_requirement(field_name: str, field: Any) -> str | None:
+    """Determine the field requirement status."""
+    if field_name not in MACHINE_GENERATED_FIELDS:
+        if field.is_required() or field_name == VARIABLE_STATUS_FIELD_NAME:
+            return REQUIRED_FIELD
+        return OPTIONAL_FIELD
+    return None
+
+
 def _populate_commented_map(
     field_name: str,
     value: str,
@@ -129,12 +141,12 @@ def _populate_commented_map(
         JsonDict,
         field.json_schema_extra,
     )[NORWEGIAN_DESCRIPTIONS]
+    field_requirement: str | None = _set_field_requirement(field_name, field)
     if description is not None:
         new_description = (
-            "\n"
-            + (REQUIRED_FIELD if field.is_required() else OPTIONAL_FIELD)
-            + "\n"
-            + str(description)
+            ("\n" + field_requirement + "\n" + str(description))
+            if field_requirement
+            else ("\n" + str(description))
         )
         commented_map.yaml_set_comment_before_after_key(
             field_name,
@@ -185,9 +197,11 @@ def configure_yaml(yaml: YAML) -> YAML:
     yaml.default_flow_style = False  # Ensures pretty YAML formatting block style
     yaml.allow_unicode = True  # Support special characters
     yaml.preserve_quotes = True
-    yaml.width = 180  # wrap long lines
+    yaml.width = 4096  # prevent wrapping lines
     yaml.indent(
-        mapping=4, sequence=2, offset=0
+        mapping=4,
+        sequence=6,
+        offset=4,
     )  # Ensure indentation for nested keys and lists
     yaml.representer.add_representer(
         VariableStatus,
@@ -209,59 +223,63 @@ def _safe_get(data: dict, keys: list):
     return data
 
 
-def _safe_set_folded(data: dict, path: str, lang: str):
-    keys = path.split(".")
-    parent = _safe_get(data, keys)
-    if isinstance(parent, dict) and lang in parent and parent[lang] is not None:
-        parent[lang] = FoldedScalarString(parent[lang])
+def _apply_literal_scalars(field: dict):
+    """Helper function to wrap `LanguageStringType` values in `LiteralScalarString`.
+
+    This function wraps each non-`None` language value in a `LanguageStringType` field
+    in the `LiteralScalarString` YAML type, ensuring proper YAML formatting with block style.
+    """
+    for lang, value in field.items():
+        if value is not None:
+            field[lang] = LiteralScalarString(value)
+
+
+def _apply_double_quotes_to_dict_values(field: dict):
+    """Helper function to wrap dictionary values in `DoubleQuotedScalarString`.
+
+    This function wraps each non-`None` value in a dictionary, including values inside lists,
+    in the `DoubleQuotedScalarString` YAML type, ensuring proper YAML formatting with double quotes.
+    """
+    for sub_key, sub_value in field.items():
+        if isinstance(sub_value, list):
+            field[sub_key] = [
+                DoubleQuotedScalarString(item) for item in sub_value if item is not None
+            ]
+        elif sub_value is not None:
+            field[sub_key] = DoubleQuotedScalarString(sub_value)
 
 
 def pre_process_data(data: dict) -> dict:
-    """Format Variable definition model fields with ruamel yaml scalar string types."""
-    folded_fields = [
-        ("definition", ["nb", "nn", "en"]),
-        ("name", ["nb", "nn", "en"]),
-        ("comment", ["nb", "nn", "en"]),
-        ("contact.title", ["nb", "nn", "en"]),
-    ]
-    for field_path, langs in folded_fields:
-        for lang in langs:
-            _safe_set_folded(data, field_path, lang)
-
-    list_fields = [
-        "unit_types",
-        "subject_fields",
-        "related_variable_definition_uris",
-    ]
-    for key in list_fields:
-        if isinstance(data.get(key), list):
+    """Format variable definition model fields with ruamel YAML scalar string types.
+
+    This method sets the appropriate scalar string type (either `LiteralScalarString` or `DoubleQuotedScalarString`)
+    for fields of the variable definition model, based on predefined lists of fields.
+
+    It processes both nested dictionaries and lists, ensuring each element is formatted with the correct YAML string type.
+
+    Args:
+        data (dict): A dictionary containing the variable definition data.
+
+    Returns:
+        dict: The updated dictionary with model fields formatted as ruamel.yaml scalar string types.
+    """
+    for key in BLOCK_FIELDS:
+        keys = key.split(".")
+        field = _safe_get(data, keys)
+        if isinstance(field, dict):
+            _apply_literal_scalars(field)
+
+    for key in DOUBLE_QUOTE_FIELDS:
+        keys = key.split(".")
+        field = _safe_get(data, keys)
+        if isinstance(field, list):
             data[key] = [
-                DoubleQuotedScalarString(item) for item in data[key] if item is not None
+                DoubleQuotedScalarString(item) for item in field if item is not None
             ]
-
-    single_line_fields = [
-        "short_name",
-        "classification_reference",
-        "measurement_type",
-        "external_reference_uri",
-        "created_by",
-        "id",
-        "last_updated_by",
-    ]
-    for key in single_line_fields:
-        if data.get(key) is not None:
+        elif isinstance(field, str):
             data[key] = DoubleQuotedScalarString(data[key])
-    # Special case due to complex structure
-    owner = data.get("owner")
-    if isinstance(owner, dict):
-        if owner.get("team") is not None:
-            owner["team"] = DoubleQuotedScalarString(owner["team"])
-        if isinstance(owner.get("groups"), list):
-            owner["groups"] = [
-                DoubleQuotedScalarString(item)
-                for item in owner["groups"]
-                if item is not None
-            ]
+        elif isinstance(field, dict):
+            _apply_double_quotes_to_dict_values(field)
 
     return data
 
@@ -322,8 +340,6 @@ def _model_to_yaml_with_comments(
                 model_instance,
             )
         elif field_name not in {VARIABLE_STATUS_FIELD_NAME, OWNER_FIELD_NAME}:
-            if isinstance(value, str):
-                value.strip()
             _populate_commented_map(field_name, value, commented_map, model_instance)
 
     base_path = (

{dapla_toolbelt_metadata-0.6.3 → dapla_toolbelt_metadata-0.6.5}/src/dapla_metadata/variable_definitions/_utils/template_files.py

@@ -41,35 +41,24 @@ def _get_default_template() -> "VariableDefinition":
 
     return VariableDefinition(
         name=LanguageStringType(
-            nb="navn",
-            nn="namn",
-            en="name",
+            nb="Navn",
         ),
         short_name="generert_kortnavn",
         definition=LanguageStringType(
-            nb="definisjonstekst",
-            nn="definisjonstekst",
-            en="definition text",
+            nb="Definisjonstekst",
         ),
         valid_from=DEFAULT_DATE,
         unit_types=[""],
         subject_fields=[""],
         contains_special_categories_of_personal_data=False,
-        variable_status=VariableStatus.DRAFT.value,
         owner=Owner(team="default team", groups=["default group"]),
         contact=Contact(
             title=LanguageStringType(
                 nb="generert tittel",
-                nn="generert tittel",
-                en="generert title",
             ),
             email="generert@ssb.no",
         ),
-        comment=LanguageStringType(
-            nb="",
-            nn="",
-            en="",
-        ),
+        variable_status=VariableStatus.DRAFT.value,
         id="",
         patch_id=0,
         created_at=DEFAULT_DATE,

{dapla_toolbelt_metadata-0.6.3 → dapla_toolbelt_metadata-0.6.5}/src/dapla_metadata/variable_definitions/_utils/variable_definition_files.py

@@ -3,6 +3,7 @@
 import logging
 from os import PathLike
 from pathlib import Path
+from typing import Any
 from typing import TypeVar
 
 from pydantic import BaseModel
@@ -17,6 +18,7 @@ from dapla_metadata.variable_definitions._utils.files import _get_current_time
 from dapla_metadata.variable_definitions._utils.files import (
     _model_to_yaml_with_comments,
 )
+from dapla_metadata.variable_definitions._utils.files import configure_yaml
 
 logger = logging.getLogger(__name__)
 
@@ -46,13 +48,38 @@ def create_variable_yaml(
 
 def _read_variable_definition_file(file_path: Path) -> dict:
     yaml = YAML()
-
+    configure_yaml(yaml)
     logger.debug("Full path to variable definition file %s", file_path)
     logger.info("Reading from '%s'", file_path.name)
     with file_path.open(encoding="utf-8") as f:
         return yaml.load(f)
 
 
+def _strip_strings_recursively(data: Any) -> Any:
+    """Recursively strip leading and trailing whitespace from string values in nested dicts/lists.
+
+    This function traverses the provided data, which may be a dictionary, list, or other types,
+    and applies the following logic:
+        - If the data is a dictionary, it recursively strips string values in all key-value pairs.
+        - If the data is a list, it recursively strips string values in all list elements.
+        - If the data is a string, it strips leading and trailing whitespace.
+        - Any other data types are returned unchanged.
+
+    Args:
+        data: The input data, which may include nested dictionaries, lists, or other types.
+
+    Returns:
+        Any: The processed data, with strings stripped of whitespace or unchanged if not a string.
+    """
+    if isinstance(data, dict):
+        return {k: _strip_strings_recursively(v) for k, v in data.items()}
+    if isinstance(data, list):
+        return [_strip_strings_recursively(item) for item in data]
+    if isinstance(data, str):
+        return data.strip()
+    return data
+
+
 def _read_file_to_model(
     file_path: PathLike[str] | None,
     model_class: type[T],
@@ -80,14 +107,14 @@ def _read_file_to_model(
         raise FileNotFoundError(
             msg,
         ) from e
+    raw_data = _read_variable_definition_file(file_path)
+    cleaned_data = _strip_strings_recursively(raw_data)
+
     model = model_class.from_dict(  # type:ignore [attr-defined]
-        _read_variable_definition_file(
-            file_path,
-        ),
+        cleaned_data
     )
 
     if model is None:
         msg = f"Could not read data from {file_path}"
         raise FileNotFoundError(msg)
-
     return model

{dapla_toolbelt_metadata-0.6.3 → dapla_toolbelt_metadata-0.6.5}/src/dapla_metadata/variable_definitions/resources/vardef_model_descriptions_nb.yaml

@@ -1,66 +1,94 @@
 # --- Variabel definisjoner ---
 # ref: https://statistics-norway.atlassian.net/wiki/spaces/MPD/pages/3009839199/VarDef+-+Krav+til+dokumentasjon+av+variabler
 name: |
-  Variabelens navn. Dette skal ikke være en mer “teknisk” forkortelse, men et navn som er forståelig for mennesker, f.eks. “Lønnsinntekter”.
+  Variabelens navn. Dette skal ikke være en mer “teknisk” forkortelse, men et navn som er forståelig for mennesker.
+  -------------------------
+  >>> EKSEMPEL:
+  name:
+      nb: |-
+          Lønnsinntekter
 short_name: |
   Dette er variabelens kortnavn, som kan være en mer “teknisk” forkortelse, f.eks. wlonn (kortnavnet til Lønnsinntekter). Kortnavnet til en variabel i Vardef skal være unikt.
-  Kravet til kortnavnet er at det kan inneholde a-z (kun små bokstaver), 0-9 og _ (understrek). Minimumslengden på kortnavnet er 2 tegn.  Bokstavene “æ”, “ø” og “å” kan ikke brukes. Disse anbefales erstattet med hhv. “ae”, “oe” og “aa"
+  Kravet til kortnavnet er at det kan inneholde a-z (kun små bokstaver), 0-9 og _ (understrek). Minimumslengden på kortnavnet er 2 tegn.
+  Bokstavene “æ”, “ø” og “å” kan ikke brukes. Disse anbefales erstattet med hhv. “ae”, “oe” og “aa"
 definition: |
   En definisjon skal beskrive hva variabelen betyr og være så kort og presis som mulig. Mer utfyllende opplysninger kan legges i Merknad-feltet.
+  -------------------------
+  >>> EKSEMPEL:
+  definition:
+      nb: |-
+          Yrkesinntekter, kapitalinntekter, skattepliktige og skattefrie overføringer, i løpet av kalenderåret.
 classification_reference: |
   ID av en klassifikasjon eller kodeliste fra KLASS som beskriver verdiene variabelen kan anta.
   For eksempel vil variabelen 'Sivilstand' ha klassifikasjon 'Standard for sivilstand' (kan vises på https://www.ssb.no/klass/klassifikasjoner/19 ) som har ID 19.
-  Eksempel: "19"
+  -------------------------
+  >>> EKSEMPEL:
+  classification_reference: "19"
 unit_types: |
-  Enhetstyper - enhetene som beskrives av denne variabelen. Variabelen “sivilstand” vil f.eks. ha enhetstypen person, mens f.eks. “Produsentpris for tjenester” vil ha både foretak og bedrift som enhetstyper siden variabelen kan beskrive begge.
+  Enhetstyper - enhetene som beskrives av denne variabelen. Variabelen “sivilstand” vil f.eks. ha enhetstypen person,
+  mens f.eks. “Produsentpris for tjenester” vil ha både foretak og bedrift som enhetstyper siden variabelen kan beskrive begge.
   Verdier skal være koder fra: https://www.ssb.no/klass/klassifikasjoner/702.
-  Eksempel:
-    - "20"
+  -------------------------
+  >>> EKSEMPEL:
+  unit_types:
+      - "20"
 subject_fields: |
   Statistikkområder som variabelen brukes innenfor. For eksempel tilhører variabelen “Sivilstand” statistikkområdet “Befolkning”.
   Verdier skal være koder fra https://www.ssb.no/klass/klassifikasjoner/618.
-  Eksempel:
-    - "bf"
-    - "be"
+  -------------------------
+  >>> EKSEMPEL:
+  subject_fields:
+      - "bf"
+      - "be"
 contains_special_categories_of_personal_data: |
   Viser om variabelen inneholder spesielt sensitive personopplysninger.
-  Kategorier:
-    - opplysninger om etnisk opprinnelse
-    - opplysninger om politisk oppfatning
-    - opplysninger om religion
-    - opplysninger om filosofisk overbevisning
-    - opplysninger om fagforeningsmedlemskap
-    - genetiske opplysninger
-    - biometriske opplysninger med det formål å entydig identifisere noen
-    - helseopplysninger
-    - opplysninger om seksuelle forhold
-    - opplysninger om seksuell legning
-  ref: https://lovdata.no/dokument/NL/lov/2018-06-15-38/KAPITTEL_gdpr-2#gdpr/a9
-  Eksempel: true
+  -------------------------
+  >>> EKSEMPEL:
+  contains_special_categories_of_personal_data: true
 measurement_type: |
   Måletype som en kvantitativ variabelen tilhører, f.eks.  valuta, areal osv.
   Verdien skal være en kode fra: https://www.ssb.no/klass/klassifikasjoner/303
-  Eksempel: "03"
+  -------------------------
+  >>> EKSEMPEL:
+  measurement_type: "03"
 valid_from: |
   Datoen variabeldefinisjonen er gyldig f.o.m.
-  Eksempel: 1999-01-11
+  -------------------------
+  >>> EKSEMPEL:
+  valid_from: 1999-01-30
 valid_until: |
   Datoen variabeldefinisjonens var gyldig t.o.m. Settes hvis definisjonen skal erstattet av en ny definisjon (med en ny gyldighetsperiode), eller variabelen ikke lenger skal brukes.
-  Eksempel: 2025-10-03
+  -------------------------
+  >>> EKSEMPEL:
+  valid_until: 2024-10-23
 external_reference_uri:  |
   En peker (URI) til ekstern definisjon/dokumentasjon, f.eks. ei webside som er relevant for variabelen.
-  Eksempel: "https://www.landbruksdirektoratet.com"
+  -----------------------------------------------------
+  >>> EKSEMPEL:
+  external_reference_uri: "https://www.landbruksdirektoratet.com"
 comment: |
-  Her kan en sette inn eventuelle tilleggsopplysninger som ikke hører hjemme i selve definisjonen. Variabelen “Landbakgrunn” har f.eks. merknaden “Fra og med 1.1.2003 ble definisjon endret til også å trekke inn besteforeldrenes fødeland”.
+  Her kan en sette inn eventuelle tilleggsopplysninger som ikke hører hjemme i selve definisjonen.
+  Variabelen “Landbakgrunn” har f.eks. merknaden “Fra og med 1.1.2003 ble definisjon endret til også å trekke inn besteforeldrenes fødeland”.
+  -----------------------------------------------------------------------------------------------
+  >>> EKSEMPEL:
+  comment:
+      nb: |-
+          Fra og med 1.1.2003 ble definisjon endret til også å trekke inn besteforeldrenes fødeland.
 related_variable_definition_uris: |
-  Her kan en legge inn URIer til andre variabler som er relevante. Eksempelvis er variabelen “Inntekt etter skatt” en beregnet variabel der  “Yrkesinntekter” og “Kapitalinntekter” inngår i beregningen. En kan da legge inn deres URI-er i dette feltet.
-  Eksempel: "https://example.com/"
+  Her kan en legge inn URIer til andre variabler som er relevante. Eksempelvis er variabelen “Inntekt etter skatt” en beregnet variabel der  “Yrkesinntekter” og “Kapitalinntekter” inngår i beregningen.
+  En kan da legge inn deres URI-er i dette feltet.
+  -------------------------
+  >>> EKSEMPEL:
+  related_variable_definition_uris:
+      - "https://example.com/"
 contact: |
   Her dokumenterer en navn og epost for person eller gruppe som kan svare på spørsmål.
-  Eksempel:
-    contact:
+  -------------------------
+  >>> EKSEMPEL:
+  contact:
       title:
-        nb: "Seksjonsleder"
+          nb: |-
+              Seksjonsleder
       email: leder@ssb.no
 variable_status: |
   Livssyklus for variabelen.