dapla-toolbelt-metadata 0.9.5__tar.gz → 0.9.6.dev1759398171__tar.gz

{dapla_toolbelt_metadata-0.9.5 → dapla_toolbelt_metadata-0.9.6.dev1759398171}/.github/workflows/release.yml

@@ -5,6 +5,10 @@ on:
     branches:
       - main
       - master
+      - release/0.9.6
+    tags:
+      - 'v*'
+
 
 jobs:
   release:
@@ -54,7 +58,6 @@ jobs:
           uv build
 
       - name: Publish package on PyPI
-        if: steps.check-version.outputs.tag
         uses: pypa/gh-action-pypi-publish@v1.13.0
 
       - name: Publish package on TestPyPI

{dapla_toolbelt_metadata-0.9.5 → dapla_toolbelt_metadata-0.9.6.dev1759398171}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dapla-toolbelt-metadata
-Version: 0.9.5
+Version: 0.9.6.dev1759398171
 Summary: Dapla Toolbelt Metadata
 Project-URL: homepage, https://github.com/statisticsnorway/dapla-toolbelt-metadata
 Project-URL: repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata

{dapla_toolbelt_metadata-0.9.5 → dapla_toolbelt_metadata-0.9.6.dev1759398171}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "dapla-toolbelt-metadata"
 description = "Dapla Toolbelt Metadata"
 license = "MIT"
-version = "0.9.5"
+version = "0.9.6.dev.1759398171"
 classifiers = ["Development Status :: 4 - Beta"]
 readme = "README.md"
 authors = [{ name = "Statistics Norway", email = "metadata@ssb.no" }]

{dapla_toolbelt_metadata-0.9.5 → dapla_toolbelt_metadata-0.9.6.dev1759398171}/src/dapla_metadata/datasets/core.py

@@ -33,10 +33,7 @@ from dapla_metadata.datasets.utility.constants import (
 from dapla_metadata.datasets.utility.constants import METADATA_DOCUMENT_FILE_SUFFIX
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
-from dapla_metadata.datasets.utility.urn import convert_definition_uris_to_urns
 from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
-from dapla_metadata.datasets.utility.utils import VariableListType
-from dapla_metadata.datasets.utility.utils import VariableType
 from dapla_metadata.datasets.utility.utils import calculate_percentage
 from dapla_metadata.datasets.utility.utils import derive_assessment_from_state
 from dapla_metadata.datasets.utility.utils import get_timestamp_now
@@ -111,8 +108,8 @@ class Datadoc:
         self.container: all_optional_model.MetadataContainer | None = None
         self.dataset_path: pathlib.Path | CloudPath | None = None
         self.dataset = all_optional_model.Dataset()
-        self.variables: VariableListType = []
-        self.variables_lookup: dict[str, VariableType] = {}
+        self.variables: list = []
+        self.variables_lookup: dict[str, all_optional_model.Variable] = {}
         self.explicitly_defined_metadata_document = False
         self.dataset_consistency_status: list[DatasetConsistencyStatus] = []
         if metadata_document_path:
@@ -207,21 +204,22 @@ class Datadoc:
         else:
             self._set_metadata(existing_metadata or extracted_metadata)
 
+        set_default_values_variables(self.variables)
+        set_default_values_dataset(cast("all_optional_model.Dataset", self.dataset))
+        set_dataset_owner(self.dataset)
+        self._create_variables_lookup()
+
     def _set_metadata(
         self,
-        metadata: OptionalDatadocMetadataType,
+        merged_metadata: OptionalDatadocMetadataType,
     ) -> None:
-        if not metadata or not (metadata.dataset and metadata.variables):
+        if not merged_metadata or not (
+            merged_metadata.dataset and merged_metadata.variables
+        ):
             msg = "Could not read metadata"
             raise ValueError(msg)
-        self.dataset = cast("all_optional_model.Dataset", metadata.dataset)
-        self.variables = metadata.variables
-
-        set_default_values_variables(self.variables)
-        set_default_values_dataset(cast("all_optional_model.Dataset", self.dataset))
-        set_dataset_owner(self.dataset)
-        convert_definition_uris_to_urns(self.variables)
-        self._create_variables_lookup()
+        self.dataset = cast("all_optional_model.Dataset", merged_metadata.dataset)
+        self.variables = merged_metadata.variables
 
     def _create_variables_lookup(self) -> None:
         self.variables_lookup = {

{dapla_toolbelt_metadata-0.9.5 → dapla_toolbelt_metadata-0.9.6.dev1759398171}/src/dapla_metadata/datasets/utility/utils.py

@@ -121,7 +121,7 @@ def derive_assessment_from_state(state: DataSetState) -> Assessment:
             return Assessment.SENSITIVE
 
 
-def set_default_values_variables(variables: VariableListType) -> None:
+def set_default_values_variables(variables: list) -> None:
     """Set default values on variables.
 
     Args:

{dapla_toolbelt_metadata-0.9.5 → dapla_toolbelt_metadata-0.9.6.dev1759398171}/uv.lock

@@ -525,7 +525,7 @@ wheels = [
 
 [[package]]
 name = "dapla-toolbelt-metadata"
-version = "0.9.4"
+version = "0.9.3"
 source = { editable = "." }
 dependencies = [
     { name = "arrow" },

{dapla_toolbelt_metadata-0.9.5 → dapla_toolbelt_metadata-0.9.6.dev1759398171}/vardef-maintenance/vardef-maintenance.toml

@@ -3,7 +3,7 @@ name = "vardef-maintenance"
 version = "0.1.0"
 description = "Dependencies necessary to run Notebooks for maintenance of variable definitions at Statistics Norway."
 requires-python = "^3.10"                                                                                             # This should match the range in dapla-toolbelt-metadata
-dependencies = ["dapla-toolbelt-metadata>=0.9.5", "ipykernel"]
+dependencies = ["dapla-toolbelt-metadata>=0.9.1", "ipykernel"]
 
 [tool.poetry]
 package-mode = false # We don't want to install this "project", just the dependencies

dapla_toolbelt_metadata-0.9.5/src/dapla_metadata/datasets/utility/urn.py

@@ -1,129 +0,0 @@
-"""Validate, parse and render URNs."""
-
-import logging
-import re
-from dataclasses import dataclass
-from enum import Enum
-from enum import auto
-
-from pydantic import AnyUrl
-
-from dapla_metadata.datasets.utility.utils import VariableListType
-
-logger = logging.getLogger(__name__)
-
-
-VARDEF_URL_TEMPLATE = "https://{subdomain}.{domain}/variable-definitions"
-
-
-class SsbNaisDomains(str, Enum):
-    """The available domains on SSBs Nais instance."""
-
-    TEST_EXTERNAL = "test.ssb.no"
-    TEST_INTERNAL = "intern.test.ssb.no"
-    PROD_EXTERNAL = "ssb.no"
-    PROD_INTERNAL = "intern.ssb.no"
-
-
-class ReferenceUrlTypes(Enum):
-    """The general category of the URL.
-
-    This can be useful to refer to when constructing a URL from a URN for a
-    specific context.
-    """
-
-    API = auto()
-    FRONTEND = auto()
-
-
-@dataclass
-class UrnConverter:
-    """Converts URLs to URNs and vice versa.
-
-    Fields:
-        urn_base: The format for the URN, up to the identifier.
-        id_pattern: A capturing group pattern which matches identifiers for this resource.
-        url_bases: The list of all the different URL representations for a resource. There
-            will typically be a number of URL representations for a particular resource,
-            depending on which system or technology they are accessed through and other
-            technical factors. This list defines which concrete URLs can be considered
-            equivalent to a URN.
-    """
-
-    urn_base: str
-    id_pattern: str
-    url_bases: list[tuple[ReferenceUrlTypes, str]]
-
-    def _extract_id(self, url: str, pattern: re.Pattern[str]) -> str | None:
-        if match := pattern.match(url):
-            return match.group(1)
-        return None
-
-    def _build_pattern(self, url_base: str) -> re.Pattern[str]:
-        return re.compile(f"^{url_base}/{self.id_pattern}")
-
-    def build_urn(self, identifier: str) -> str:
-        """Build a URN for the given identifier."""
-        return f"{self.urn_base}:{identifier}"
-
-    def convert_to_urn(self, url: str | AnyUrl | None) -> str | None:
-        """Convert a URL to a generalized URN for that same resource.
-
-        Args:
-            url (str | None): The URL to convert.
-
-        Returns:
-            str | None: The URN or None if it can't be converted.
-        """
-        if not url:
-            return None
-
-        patterns = (self._build_pattern(url[-1]) for url in self.url_bases)
-        matches = (self._extract_id(str(url), p) for p in patterns)
-        identifier = next((m for m in matches if m), None)
-        if identifier:
-            return self.build_urn(identifier)
-
-        return None
-
-
-vardef_urn_converter = UrnConverter(
-    urn_base="urn:ssb:variable-definition:vardef",
-    id_pattern=r"([a-z0-9]{8})",
-    url_bases=[
-        *[
-            (
-                ReferenceUrlTypes.API,
-                VARDEF_URL_TEMPLATE.format(
-                    subdomain="metadata", domain=nais_domain.value
-                ),
-            )
-            for nais_domain in SsbNaisDomains
-        ],
-        *[
-            (
-                ReferenceUrlTypes.FRONTEND,
-                VARDEF_URL_TEMPLATE.format(
-                    subdomain="catalog", domain=nais_domain.value
-                ),
-            )
-            for nais_domain in SsbNaisDomains
-        ],
-    ],
-)
-
-
-def convert_definition_uris_to_urns(variables: VariableListType) -> None:
-    """Where definition URIs are recognized URLs, convert them to URNs.
-
-    Where the value is not a known URL we preserve the value as it is and log an
-    ERROR level message.
-
-    Args:
-        variables (VariableListType): The list of variables.
-    """
-    for v in variables:
-        if urn := vardef_urn_converter.convert_to_urn(v.definition_uri):
-            v.definition_uri = urn  # type: ignore [assignment]
-        else:
-            logger.error("Could not convert value to URN: %s", v.definition_uri)

dapla_toolbelt_metadata-0.9.5/tests/datasets/test_urn.py

@@ -1,69 +0,0 @@
-import logging
-
-import datadoc_model.all_optional.model as all_optional_model
-import pytest
-
-from dapla_metadata.datasets.core import Datadoc
-from dapla_metadata.datasets.utility.urn import SsbNaisDomains
-from dapla_metadata.datasets.utility.urn import vardef_urn_converter
-
-VARIABLE_DEFINITION_URN_TEST_CASES = [
-    (None, None),
-    *[
-        (
-            f"https://metadata.{domain.value}/variable-definitions/hd8sks89",
-            "urn:ssb:variable-definition:vardef:hd8sks89",
-        )
-        for domain in SsbNaisDomains
-    ],
-    *[
-        (
-            f"https://catalog.{domain.value}/variable-definitions/hd8sks89",
-            "urn:ssb:variable-definition:vardef:hd8sks89",
-        )
-        for domain in SsbNaisDomains
-    ],
-]
-
-
-@pytest.mark.parametrize(
-    ("case", "expected_result"),
-    [
-        *VARIABLE_DEFINITION_URN_TEST_CASES,
-        (
-            "https://www.vg.no",
-            None,
-        ),
-    ],
-)
-def test_convert_to_vardef_urn(case: str | None, expected_result: str | None):
-    assert vardef_urn_converter.convert_to_urn(case) == expected_result
-
-
-@pytest.mark.parametrize(
-    ("case", "expected_result"),
-    [
-        *VARIABLE_DEFINITION_URN_TEST_CASES,
-        (
-            "https://www.vg.no",
-            "https://www.vg.no/",
-        ),
-    ],
-)
-def test_convert_to_vardef_urn_end_to_end(
-    case: str | None, expected_result: str | None, caplog
-):
-    caplog.set_level(logging.ERROR)
-    meta = Datadoc()
-    meta._set_metadata(  # noqa: SLF001
-        all_optional_model.DatadocMetadata(
-            dataset=all_optional_model.Dataset(),
-            variables=[all_optional_model.Variable(definition_uri=case)],
-        )
-    )
-    if not expected_result:
-        assert meta.variables[0].definition_uri is None
-    else:
-        assert str(meta.variables[0].definition_uri) == expected_result
-        if not expected_result.startswith("urn:"):
-            assert "Could not convert value to URN" in caplog.text