dapla-toolbelt-metadata 0.6.2__tar.gz → 0.6.3__tar.gz

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dapla-toolbelt-metadata
-Version: 0.6.2
+Version: 0.6.3
 Summary: Dapla Toolbelt Metadata
 License: MIT
 Author: Team Metadata

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dapla-toolbelt-metadata"
-version = "0.6.2"
+version = "0.6.3"
 description = "Dapla Toolbelt Metadata"
 authors = ["Team Metadata <metadata@ssb.no>"]
 license = "MIT"

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/datasets/core.py

@@ -111,6 +111,7 @@ class Datadoc:
         self.variables: list = []
         self.variables_lookup: dict[str, model.Variable] = {}
         self.explicitly_defined_metadata_document = False
+        self.dataset_consistency_status: list = []
         if metadata_document_path:
             self.metadata_document = normalize_path(metadata_document_path)
             self.explicitly_defined_metadata_document = True
@@ -169,11 +170,14 @@ class Datadoc:
             and existing_metadata is not None
         ):
             existing_file_path = self._get_existing_file_path(extracted_metadata)
-            self._check_ready_to_merge(
+            self.dataset_consistency_status = self._check_dataset_consistency(
                 self.dataset_path,
                 Path(existing_file_path),
                 extracted_metadata,
                 existing_metadata,
+            )
+            self._check_ready_to_merge(
+                self.dataset_consistency_status,
                 errors_as_warnings=self.errors_as_warnings,
             )
             merged_metadata = self._merge_metadata(
@@ -224,29 +228,26 @@ class Datadoc:
         }
 
     @staticmethod
-    def _check_ready_to_merge(
+    def _check_dataset_consistency(
         new_dataset_path: Path | CloudPath,
         existing_dataset_path: Path,
         extracted_metadata: model.DatadocMetadata,
         existing_metadata: model.DatadocMetadata,
-        *,
-        errors_as_warnings: bool,
-    ) -> None:
-        """Check if the datasets are consistent enough to make a successful merge of metadata.
+    ) -> list[dict[str, object]]:
+        """Run consistency tests.
 
         Args:
             new_dataset_path: Path to the dataset to be documented.
             existing_dataset_path: Path stored in the existing metadata.
             extracted_metadata: Metadata extracted from a physical dataset.
             existing_metadata: Metadata from a previously created metadata document.
-            errors_as_warnings: True if failing checks should be raised as warnings, not errors.
 
-        Raises:
-            InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+        Returns:
+            List if dict with property name and boolean success flag
         """
         new_dataset_path_info = DaplaDatasetPathInfo(new_dataset_path)
         existing_dataset_path_info = DaplaDatasetPathInfo(existing_dataset_path)
-        results = [
+        return [
             {
                 "name": "Bucket name",
                 "success": (
@@ -290,6 +291,20 @@ class Datadoc:
                 ),
             },
         ]
+
+    @staticmethod
+    def _check_ready_to_merge(
+        results: list[dict[str, object]], *, errors_as_warnings: bool
+    ) -> None:
+        """Check if the datasets are consistent enough to make a successful merge of metadata.
+
+        Args:
+            results: List if dict with property name and boolean success flag
+            errors_as_warnings: True if failing checks should be raised as warnings, not errors.
+
+        Raises:
+            InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+        """
         if failures := [result for result in results if not result["success"]]:
             msg = f"{INCONSISTENCIES_MESSAGE} {', '.join(str(f['name']) for f in failures)}"
             if errors_as_warnings:

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/standards/name_validator.py

@@ -128,9 +128,9 @@ class NamingStandardReport:
         """Returns an appropriate message based on the success rate."""
         rate = self.success_rate()
         if rate is not None:
-            if rate == 100:
+            if 95 <= rate <= 100:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_BEST
-            if 70 < rate < 100:
+            if 70 < rate < 95:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_GOOD
             if 40 <= rate <= 70:
                 return SSB_NAMING_STANDARD_REPORT_RESULT_AVERAGE

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/standards/utils/constants.py

@@ -9,7 +9,7 @@ NAME_STANDARD_SUCCESS = "Filene dine er i samsvar med SSB-navnestandarden"
 NAME_STANDARD_VIOLATION = "Det er oppdaget brudd på SSB-navnestandard:"
 
 MISSING_BUCKET_NAME = "Filnavn mangler bøttenavn ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
-MISSING_VERSION = "Filnavn mangler versjon ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
+MISSING_VERSION = "Filnavn mangler versjon, hvis ikke filen er nyeste versjon kan dette være brudd på navnestandarden ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#versjonering-av-datasett"
 MISSING_PERIOD = "Filnavn mangler gyldighetsperiode ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
 MISSING_SHORT_NAME = "Kortnavn for statistikk mangler ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
 MISSING_DATA_STATE = "Mappe for datatilstand mangler ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
@@ -26,7 +26,7 @@ BUCKET_NAME_UNKNOWN = "Kan ikke validere bøttenavn"
 
 SSB_NAMING_STANDARD_REPORT = "SSB navnestandard rapport"
 SSB_NAMING_STANDARD_REPORT_SUCCESS_RATE = "Suksess rate"
-SSB_NAMING_STANDARD_REPORT_RESULT_BEST = "🚀 Fantastisk! Alt bestått! 🎉\n"
+SSB_NAMING_STANDARD_REPORT_RESULT_BEST = "🚀 Fantastisk! 🎉\n"
 SSB_NAMING_STANDARD_REPORT_RESULT_GOOD = (
     "✅ Bra jobba! Fortsatt litt rom for forbedring. 😊\n"
 )

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/variable_definitions/_utils/config.py

@@ -59,8 +59,7 @@ def get_vardef_host() -> str:
         case DaplaEnvironment.TEST:
             return VARDEF_HOST_TEST
         case DaplaEnvironment.DEV:
-            msg = "Vardef is not available in dev."
-            raise NotImplementedError(msg)
+            return VARDEF_HOST_TEST
         case _:
             return get_config_item("VARDEF_HOST") or "http://localhost:8080"
 

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/variable_definitions/_utils/constants.py

@@ -39,3 +39,5 @@ MACHINE_GENERATED_FIELDS = [
 
 OPTIONAL_FIELD = "~ Valgfritt felt ~"
 REQUIRED_FIELD = "! Obligatorisk felt !"
+
+YAML_STR_TAG = "tag:yaml.org,2002:str"

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/variable_definitions/_utils/files.py

@@ -10,6 +10,9 @@ import pytz
 from pydantic.config import JsonDict
 from ruamel.yaml import YAML
 from ruamel.yaml import CommentedMap
+from ruamel.yaml import RoundTripRepresenter
+from ruamel.yaml.scalarstring import DoubleQuotedScalarString
+from ruamel.yaml.scalarstring import FoldedScalarString
 
 from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
     CompleteResponse,
@@ -40,6 +43,7 @@ from dapla_metadata.variable_definitions._utils.constants import (
 from dapla_metadata.variable_definitions._utils.constants import (
     VARIABLE_STATUS_FIELD_NAME,
 )
+from dapla_metadata.variable_definitions._utils.constants import YAML_STR_TAG
 from dapla_metadata.variable_definitions._utils.descriptions import (
     apply_norwegian_descriptions_to_model,
 )
@@ -120,14 +124,15 @@ def _populate_commented_map(
 ) -> None:
     """Add data to a CommentedMap."""
     commented_map[field_name] = value
-    field = model_instance.model_fields[field_name]
+    field = type(model_instance).model_fields[field_name]
     description: JsonValue = cast(
         JsonDict,
         field.json_schema_extra,
     )[NORWEGIAN_DESCRIPTIONS]
     if description is not None:
         new_description = (
-            (REQUIRED_FIELD if field.is_required() else OPTIONAL_FIELD)
+            "\n"
+            + (REQUIRED_FIELD if field.is_required() else OPTIONAL_FIELD)
             + "\n"
             + str(description)
         )
@@ -174,14 +179,20 @@ def _validate_and_create_directory(custom_directory: Path) -> Path:
     return custom_directory
 
 
-def _configure_yaml() -> YAML:
-    yaml = YAML()  # Use ruamel.yaml library
-    yaml.default_flow_style = False  # Ensures pretty YAML formatting
-
+def configure_yaml(yaml: YAML) -> YAML:
+    """Common Yaml config for variable definitions."""
+    yaml.Representer = RoundTripRepresenter  # Preserve the order of keys etc.
+    yaml.default_flow_style = False  # Ensures pretty YAML formatting block style
+    yaml.allow_unicode = True  # Support special characters
+    yaml.preserve_quotes = True
+    yaml.width = 180  # wrap long lines
+    yaml.indent(
+        mapping=4, sequence=2, offset=0
+    )  # Ensure indentation for nested keys and lists
     yaml.representer.add_representer(
         VariableStatus,
         lambda dumper, data: dumper.represent_scalar(
-            "tag:yaml.org,2002:str",
+            YAML_STR_TAG,
             data.value,
         ),
     )
@@ -189,6 +200,72 @@ def _configure_yaml() -> YAML:
     return yaml
 
 
+def _safe_get(data: dict, keys: list):
+    """Safely navigate nested dictionaries."""
+    for key in keys:
+        if not isinstance(data, dict) or key not in data or data[key] is None:
+            return None
+        data = data[key]
+    return data
+
+
+def _safe_set_folded(data: dict, path: str, lang: str):
+    keys = path.split(".")
+    parent = _safe_get(data, keys)
+    if isinstance(parent, dict) and lang in parent and parent[lang] is not None:
+        parent[lang] = FoldedScalarString(parent[lang])
+
+
+def pre_process_data(data: dict) -> dict:
+    """Format Variable definition model fields with ruamel yaml scalar string types."""
+    folded_fields = [
+        ("definition", ["nb", "nn", "en"]),
+        ("name", ["nb", "nn", "en"]),
+        ("comment", ["nb", "nn", "en"]),
+        ("contact.title", ["nb", "nn", "en"]),
+    ]
+    for field_path, langs in folded_fields:
+        for lang in langs:
+            _safe_set_folded(data, field_path, lang)
+
+    list_fields = [
+        "unit_types",
+        "subject_fields",
+        "related_variable_definition_uris",
+    ]
+    for key in list_fields:
+        if isinstance(data.get(key), list):
+            data[key] = [
+                DoubleQuotedScalarString(item) for item in data[key] if item is not None
+            ]
+
+    single_line_fields = [
+        "short_name",
+        "classification_reference",
+        "measurement_type",
+        "external_reference_uri",
+        "created_by",
+        "id",
+        "last_updated_by",
+    ]
+    for key in single_line_fields:
+        if data.get(key) is not None:
+            data[key] = DoubleQuotedScalarString(data[key])
+    # Special case due to complex structure
+    owner = data.get("owner")
+    if isinstance(owner, dict):
+        if owner.get("team") is not None:
+            owner["team"] = DoubleQuotedScalarString(owner["team"])
+        if isinstance(owner.get("groups"), list):
+            owner["groups"] = [
+                DoubleQuotedScalarString(item)
+                for item in owner["groups"]
+                if item is not None
+            ]
+
+    return data
+
+
 def _model_to_yaml_with_comments(
     model_instance: CompleteResponse,
     file_name: str,
@@ -209,7 +286,8 @@ def _model_to_yaml_with_comments(
     Returns:
         Path: The file path of the generated YAML file.
     """
-    yaml = _configure_yaml()
+    yaml = YAML()
+    configure_yaml(yaml)
 
     from dapla_metadata.variable_definitions.variable_definition import (
         VariableDefinition,
@@ -223,7 +301,7 @@ def _model_to_yaml_with_comments(
         serialize_as_any=True,
         warnings="error",
     )
-
+    data = pre_process_data(data)
     # One CommentMap for each section in the yaml file
     machine_generated_map = CommentedMap()
     commented_map = CommentedMap()
@@ -244,6 +322,8 @@ def _model_to_yaml_with_comments(
                 model_instance,
             )
         elif field_name not in {VARIABLE_STATUS_FIELD_NAME, OWNER_FIELD_NAME}:
+            if isinstance(value, str):
+                value.strip()
             _populate_commented_map(field_name, value, commented_map, model_instance)
 
     base_path = (

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/variable_definitions/_utils/template_files.py

@@ -41,30 +41,34 @@ def _get_default_template() -> "VariableDefinition":
 
     return VariableDefinition(
         name=LanguageStringType(
-            nb="default navn",
-            nn="default namn",
-            en="default name",
+            nb="navn",
+            nn="namn",
+            en="name",
         ),
-        short_name="default_kortnavn",
+        short_name="generert_kortnavn",
         definition=LanguageStringType(
-            nb="default definisjon",
-            nn="default definisjon",
-            en="default definition",
+            nb="definisjonstekst",
+            nn="definisjonstekst",
+            en="definition text",
         ),
-        classification_reference="class_id",
         valid_from=DEFAULT_DATE,
-        unit_types=["00"],
-        subject_fields=["aa"],
+        unit_types=[""],
+        subject_fields=[""],
         contains_special_categories_of_personal_data=False,
         variable_status=VariableStatus.DRAFT.value,
         owner=Owner(team="default team", groups=["default group"]),
         contact=Contact(
             title=LanguageStringType(
-                nb="default tittel",
-                nn="default tittel",
-                en="default title",
+                nb="generert tittel",
+                nn="generert tittel",
+                en="generert title",
             ),
-            email="default@ssb.no",
+            email="generert@ssb.no",
+        ),
+        comment=LanguageStringType(
+            nb="",
+            nn="",
+            en="",
         ),
         id="",
         patch_id=0,

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/variable_definitions/resources/vardef_model_descriptions_nb.yaml

@@ -10,12 +10,18 @@ definition: |
 classification_reference: |
   ID av en klassifikasjon eller kodeliste fra KLASS som beskriver verdiene variabelen kan anta.
   For eksempel vil variabelen 'Sivilstand' ha klassifikasjon 'Standard for sivilstand' (kan vises på https://www.ssb.no/klass/klassifikasjoner/19 ) som har ID 19.
+  Eksempel: "19"
 unit_types: |
   Enhetstyper - enhetene som beskrives av denne variabelen. Variabelen “sivilstand” vil f.eks. ha enhetstypen person, mens f.eks. “Produsentpris for tjenester” vil ha både foretak og bedrift som enhetstyper siden variabelen kan beskrive begge.
   Verdier skal være koder fra: https://www.ssb.no/klass/klassifikasjoner/702.
+  Eksempel:
+    - "20"
 subject_fields: |
   Statistikkområder som variabelen brukes innenfor. For eksempel tilhører variabelen “Sivilstand” statistikkområdet “Befolkning”.
   Verdier skal være koder fra https://www.ssb.no/klass/klassifikasjoner/618.
+  Eksempel:
+    - "bf"
+    - "be"
 contains_special_categories_of_personal_data: |
   Viser om variabelen inneholder spesielt sensitive personopplysninger.
   Kategorier:
@@ -30,21 +36,32 @@ contains_special_categories_of_personal_data: |
     - opplysninger om seksuelle forhold
     - opplysninger om seksuell legning
   ref: https://lovdata.no/dokument/NL/lov/2018-06-15-38/KAPITTEL_gdpr-2#gdpr/a9
+  Eksempel: true
 measurement_type: |
   Måletype som en kvantitativ variabelen tilhører, f.eks.  valuta, areal osv.
   Verdien skal være en kode fra: https://www.ssb.no/klass/klassifikasjoner/303
+  Eksempel: "03"
 valid_from: |
   Datoen variabeldefinisjonen er gyldig f.o.m.
+  Eksempel: 1999-01-11
 valid_until: |
   Datoen variabeldefinisjonens var gyldig t.o.m. Settes hvis definisjonen skal erstattet av en ny definisjon (med en ny gyldighetsperiode), eller variabelen ikke lenger skal brukes.
+  Eksempel: 2025-10-03
 external_reference_uri:  |
   En peker (URI) til ekstern definisjon/dokumentasjon, f.eks. ei webside som er relevant for variabelen.
+  Eksempel: "https://www.landbruksdirektoratet.com"
 comment: |
   Her kan en sette inn eventuelle tilleggsopplysninger som ikke hører hjemme i selve definisjonen. Variabelen “Landbakgrunn” har f.eks. merknaden “Fra og med 1.1.2003 ble definisjon endret til også å trekke inn besteforeldrenes fødeland”.
 related_variable_definition_uris: |
   Her kan en legge inn URIer til andre variabler som er relevante. Eksempelvis er variabelen “Inntekt etter skatt” en beregnet variabel der  “Yrkesinntekter” og “Kapitalinntekter” inngår i beregningen. En kan da legge inn deres URI-er i dette feltet.
+  Eksempel: "https://example.com/"
 contact: |
   Her dokumenterer en navn og epost for person eller gruppe som kan svare på spørsmål.
+  Eksempel:
+    contact:
+      title:
+        nb: "Seksjonsleder"
+      email: leder@ssb.no
 variable_status: |
   Livssyklus for variabelen.
 id: |

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/variable_definitions/vardef.py

@@ -108,7 +108,7 @@ class Vardef:
         )
 
         logger.info(
-            "Successfully created variable definition '%s' with ID '%s'",
+            "✅ Successfully created variable definition '%s' with ID '%s'",
             new_variable.short_name,
             new_variable.id,
         )
@@ -165,7 +165,7 @@ class Vardef:
         )
 
         logger.info(
-            "Successfully migrated variable definition '%s' with ID '%s'",
+            "✅ Successfully migrated variable definition '%s' with ID '%s'",
             migrated_variable.short_name,
             migrated_variable.id,
         )
@@ -273,7 +273,7 @@ class Vardef:
             custom_directory=Path(custom_file_path) if custom_file_path else None,
         )
         logger.info(
-            f"Created editable variable definition template file at {file_path}",  # noqa: G004
+            f"✅ Created editable variable definition template file at {file_path}",  # noqa: G004
         )
         return file_path
 

{dapla_toolbelt_metadata-0.6.2 → dapla_toolbelt_metadata-0.6.3}/src/dapla_metadata/variable_definitions/variable_definition.py

@@ -34,6 +34,8 @@ from dapla_metadata.variable_definitions._generated.vardef_client.models.variabl
 )
 from dapla_metadata.variable_definitions._utils import config
 from dapla_metadata.variable_definitions._utils._client import VardefClient
+from dapla_metadata.variable_definitions._utils.files import configure_yaml
+from dapla_metadata.variable_definitions._utils.files import pre_process_data
 from dapla_metadata.variable_definitions._utils.variable_definition_files import (
     _read_file_to_model,
 )
@@ -126,7 +128,7 @@ class VariableDefinition(CompleteResponse):
         self.__dict__.update(updated)
 
         logger.info(
-            "Successfully updated variable definition '%s' with ID '%s'",
+            "✅ Successfully updated variable definition '%s' with ID '%s'",
             updated.short_name,
             updated.id,
         )
@@ -176,7 +178,7 @@ class VariableDefinition(CompleteResponse):
             variable_definition_id=self.id,
             active_group=config.get_active_group(),
         )
-        return f"Variable {self.id} safely deleted"
+        return f"✅ Variable {self.id} safely deleted"
 
     @vardef_exception_handler
     def get_patch(self, patch_id: int) -> "VariableDefinition":
@@ -205,9 +207,9 @@ class VariableDefinition(CompleteResponse):
 
         Patches are to be used for minor changes which don't require a new Validity Period.
         Examples of reasons for creating a new Patch:
-          - Correcting a typo
-          - Adding a translation
-          - Adding a subject field
+        - Correcting a typo
+        - Adding a translation
+        - Adding a subject field
 
         Supply only the fields to be changed. Other fields will retain their current values.
 
@@ -234,7 +236,7 @@ class VariableDefinition(CompleteResponse):
         self.__dict__.update(new_patch)
 
         logger.info(
-            "Successfully created patch with patch ID '%s' for variable definition '%s' with ID '%s'",
+            "✅ Successfully created patch with patch ID '%s' for variable definition '%s' with ID '%s'",
             new_patch.patch_id,
             new_patch.short_name,
             new_patch.id,
@@ -309,7 +311,7 @@ class VariableDefinition(CompleteResponse):
         self.__dict__.update(new_validity_period)
 
         logger.info(
-            "Successfully created validity period that is valid from '%s' for variable definition '%s' with ID '%s'",
+            "✅ Successfully created validity period that is valid from '%s' for variable definition '%s' with ID '%s'",
             new_validity_period.valid_from,
             new_validity_period.short_name,
             new_validity_period.id,
@@ -350,7 +352,7 @@ class VariableDefinition(CompleteResponse):
             UpdateDraft(variable_status=VariableStatus.PUBLISHED_INTERNAL),
         )
         logger.info(
-            "Variable definition '%s' with ID '%s' successfully published, new status: %s",
+            "✅ Variable definition '%s' with ID '%s' successfully published, new status: %s",
             update.short_name,
             update.id,
             update.variable_status,
@@ -373,7 +375,7 @@ class VariableDefinition(CompleteResponse):
                 Patch(variable_status=VariableStatus.PUBLISHED_EXTERNAL),
             )
         logger.info(
-            "Variable definition '%s' with ID '%s' successfully published, new status: %s",
+            "✅ Variable definition '%s' with ID '%s' successfully published, new status: %s",
             update.short_name,
             update.id,
             update.variable_status,
@@ -387,7 +389,7 @@ class VariableDefinition(CompleteResponse):
         )
         self.set_file_path(file_path)
         logger.info(
-            f"Created editable variable definition file at {file_path}",  # noqa: G004
+            f"✅ Created editable variable definition file at {file_path}",  # noqa: G004
         )
         return self
 
@@ -405,16 +407,12 @@ class VariableDefinition(CompleteResponse):
 
     def _convert_to_yaml_output(self) -> str:
         stream = StringIO()
-        with ruamel.yaml.YAML(
-            output=stream,
-        ) as yaml:
-            yaml.default_flow_style = False
-            yaml.allow_unicode = True
-            yaml.dump(
-                self.model_dump(
-                    mode="json",
-                    serialize_as_any=True,
-                    warnings="error",
-                ),
+        with ruamel.yaml.YAML(output=stream) as yaml:
+            configure_yaml(yaml)
+            data = self.model_dump(
+                mode="json",
+                serialize_as_any=True,
+                warnings="error",
             )
+            yaml.dump(pre_process_data(data))
         return stream.getvalue()