dapla-toolbelt-metadata 0.9.4__py3-none-any.whl → 0.9.5__py3-none-any.whl

dapla_metadata/datasets/core.py

@@ -33,7 +33,10 @@ from dapla_metadata.datasets.utility.constants import (
 from dapla_metadata.datasets.utility.constants import METADATA_DOCUMENT_FILE_SUFFIX
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
+from dapla_metadata.datasets.utility.urn import convert_definition_uris_to_urns
 from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
+from dapla_metadata.datasets.utility.utils import VariableListType
+from dapla_metadata.datasets.utility.utils import VariableType
 from dapla_metadata.datasets.utility.utils import calculate_percentage
 from dapla_metadata.datasets.utility.utils import derive_assessment_from_state
 from dapla_metadata.datasets.utility.utils import get_timestamp_now
@@ -108,8 +111,8 @@ class Datadoc:
         self.container: all_optional_model.MetadataContainer | None = None
         self.dataset_path: pathlib.Path | CloudPath | None = None
         self.dataset = all_optional_model.Dataset()
-        self.variables: list = []
-        self.variables_lookup: dict[str, all_optional_model.Variable] = {}
+        self.variables: VariableListType = []
+        self.variables_lookup: dict[str, VariableType] = {}
         self.explicitly_defined_metadata_document = False
         self.dataset_consistency_status: list[DatasetConsistencyStatus] = []
         if metadata_document_path:
@@ -204,22 +207,21 @@ class Datadoc:
         else:
             self._set_metadata(existing_metadata or extracted_metadata)
 
-        set_default_values_variables(self.variables)
-        set_default_values_dataset(cast("all_optional_model.Dataset", self.dataset))
-        set_dataset_owner(self.dataset)
-        self._create_variables_lookup()
-
     def _set_metadata(
         self,
-        merged_metadata: OptionalDatadocMetadataType,
+        metadata: OptionalDatadocMetadataType,
     ) -> None:
-        if not merged_metadata or not (
-            merged_metadata.dataset and merged_metadata.variables
-        ):
+        if not metadata or not (metadata.dataset and metadata.variables):
             msg = "Could not read metadata"
             raise ValueError(msg)
-        self.dataset = cast("all_optional_model.Dataset", merged_metadata.dataset)
-        self.variables = merged_metadata.variables
+        self.dataset = cast("all_optional_model.Dataset", metadata.dataset)
+        self.variables = metadata.variables
+
+        set_default_values_variables(self.variables)
+        set_default_values_dataset(cast("all_optional_model.Dataset", self.dataset))
+        set_dataset_owner(self.dataset)
+        convert_definition_uris_to_urns(self.variables)
+        self._create_variables_lookup()
 
     def _create_variables_lookup(self) -> None:
         self.variables_lookup = {

dapla_metadata/datasets/utility/urn.py

@@ -0,0 +1,129 @@
+"""Validate, parse and render URNs."""
+
+import logging
+import re
+from dataclasses import dataclass
+from enum import Enum
+from enum import auto
+
+from pydantic import AnyUrl
+
+from dapla_metadata.datasets.utility.utils import VariableListType
+
+logger = logging.getLogger(__name__)
+
+
+VARDEF_URL_TEMPLATE = "https://{subdomain}.{domain}/variable-definitions"
+
+
+class SsbNaisDomains(str, Enum):
+    """The available domains on SSBs Nais instance."""
+
+    TEST_EXTERNAL = "test.ssb.no"
+    TEST_INTERNAL = "intern.test.ssb.no"
+    PROD_EXTERNAL = "ssb.no"
+    PROD_INTERNAL = "intern.ssb.no"
+
+
+class ReferenceUrlTypes(Enum):
+    """The general category of the URL.
+
+    This can be useful to refer to when constructing a URL from a URN for a
+    specific context.
+    """
+
+    API = auto()
+    FRONTEND = auto()
+
+
+@dataclass
+class UrnConverter:
+    """Converts URLs to URNs and vice versa.
+
+    Fields:
+        urn_base: The format for the URN, up to the identifier.
+        id_pattern: A capturing group pattern which matches identifiers for this resource.
+        url_bases: The list of all the different URL representations for a resource. There
+            will typically be a number of URL representations for a particular resource,
+            depending on which system or technology they are accessed through and other
+            technical factors. This list defines which concrete URLs can be considered
+            equivalent to a URN.
+    """
+
+    urn_base: str
+    id_pattern: str
+    url_bases: list[tuple[ReferenceUrlTypes, str]]
+
+    def _extract_id(self, url: str, pattern: re.Pattern[str]) -> str | None:
+        if match := pattern.match(url):
+            return match.group(1)
+        return None
+
+    def _build_pattern(self, url_base: str) -> re.Pattern[str]:
+        return re.compile(f"^{url_base}/{self.id_pattern}")
+
+    def build_urn(self, identifier: str) -> str:
+        """Build a URN for the given identifier."""
+        return f"{self.urn_base}:{identifier}"
+
+    def convert_to_urn(self, url: str | AnyUrl | None) -> str | None:
+        """Convert a URL to a generalized URN for that same resource.
+
+        Args:
+            url (str | None): The URL to convert.
+
+        Returns:
+            str | None: The URN or None if it can't be converted.
+        """
+        if not url:
+            return None
+
+        patterns = (self._build_pattern(url[-1]) for url in self.url_bases)
+        matches = (self._extract_id(str(url), p) for p in patterns)
+        identifier = next((m for m in matches if m), None)
+        if identifier:
+            return self.build_urn(identifier)
+
+        return None
+
+
+vardef_urn_converter = UrnConverter(
+    urn_base="urn:ssb:variable-definition:vardef",
+    id_pattern=r"([a-z0-9]{8})",
+    url_bases=[
+        *[
+            (
+                ReferenceUrlTypes.API,
+                VARDEF_URL_TEMPLATE.format(
+                    subdomain="metadata", domain=nais_domain.value
+                ),
+            )
+            for nais_domain in SsbNaisDomains
+        ],
+        *[
+            (
+                ReferenceUrlTypes.FRONTEND,
+                VARDEF_URL_TEMPLATE.format(
+                    subdomain="catalog", domain=nais_domain.value
+                ),
+            )
+            for nais_domain in SsbNaisDomains
+        ],
+    ],
+)
+
+
+def convert_definition_uris_to_urns(variables: VariableListType) -> None:
+    """Where definition URIs are recognized URLs, convert them to URNs.
+
+    Where the value is not a known URL we preserve the value as it is and log an
+    ERROR level message.
+
+    Args:
+        variables (VariableListType): The list of variables.
+    """
+    for v in variables:
+        if urn := vardef_urn_converter.convert_to_urn(v.definition_uri):
+            v.definition_uri = urn  # type: ignore [assignment]
+        else:
+            logger.error("Could not convert value to URN: %s", v.definition_uri)

dapla_metadata/datasets/utility/utils.py

@@ -121,7 +121,7 @@ def derive_assessment_from_state(state: DataSetState) -> Assessment:
             return Assessment.SENSITIVE
 
 
-def set_default_values_variables(variables: list) -> None:
+def set_default_values_variables(variables: VariableListType) -> None:
     """Set default values on variables.
 
     Args:

{dapla_toolbelt_metadata-0.9.4.dist-info → dapla_toolbelt_metadata-0.9.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dapla-toolbelt-metadata
-Version: 0.9.4
+Version: 0.9.5
 Summary: Dapla Toolbelt Metadata
 Project-URL: homepage, https://github.com/statisticsnorway/dapla-toolbelt-metadata
 Project-URL: repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata

{dapla_toolbelt_metadata-0.9.4.dist-info → dapla_toolbelt_metadata-0.9.5.dist-info}/RECORD

@@ -8,7 +8,7 @@ dapla_metadata/dapla/user_info.py,sha256=bENez-ICt9ySR8orYebO68Q3_2LkIW9QTL58DTc
 dapla_metadata/datasets/__init__.py,sha256=an-REJgi7N8-S1SCz-MYO_8as6fMe03WvhjRP_hWWkg,293
 dapla_metadata/datasets/_merge.py,sha256=Tk5wQz6xZGr8veUAHZb42O8HARU8ObBJ_E4afvVWdlo,12993
 dapla_metadata/datasets/code_list.py,sha256=JtCE-5Q8grAKvkn0KKjzeGhO-96O7yGsastbuoakreg,9057
-dapla_metadata/datasets/core.py,sha256=p-2OJsAEWCUqBlzn0YIYkK-pAgtvMROdoxXvCyjfWYs,20434
+dapla_metadata/datasets/core.py,sha256=GVDnzLZagTpaXin6eC1xB62U-LFinDuWVeZmyx-9pas,20633
 dapla_metadata/datasets/dapla_dataset_path_info.py,sha256=WPeV_mwKk2B9sXd14SaP-kTb1bOQ_8W2KtrqOG7sJIY,26867
 dapla_metadata/datasets/dataset_parser.py,sha256=3dtRXNy1C8SfG8zTYWdY26nV4l-dG25IC_0J5t2bYwI,8285
 dapla_metadata/datasets/model_validation.py,sha256=6qqq1ueTWRWBPTwEGJD49Pv7ksMEaq0iDtuOXelaw-s,7223
@@ -23,7 +23,8 @@ dapla_metadata/datasets/external_sources/external_sources.py,sha256=9eIcOIUbaodN
 dapla_metadata/datasets/utility/__init__.py,sha256=pp6tUcgUbo8iq9OPtFKQrTbLuI3uY7NHptwWSTpasOU,33
 dapla_metadata/datasets/utility/constants.py,sha256=YKsn6GfNIkwLoBp0yq209o0TbsEhsA_jGaZLVR984JU,2933
 dapla_metadata/datasets/utility/enums.py,sha256=i6dcxWya5k4LjLdGGIM_H37rRndizug3peaAgoE5UdM,652
-dapla_metadata/datasets/utility/utils.py,sha256=Enlmhj1BA7C9Im1ju3EwS6_kV1cpT53wb2cCtBGs_lI,20145
+dapla_metadata/datasets/utility/urn.py,sha256=UfiQZf49zbrqRwBzD9yUgfwi6sXRl3KhS9seKhhYuUs,3896
+dapla_metadata/datasets/utility/utils.py,sha256=q76UJI8W4j2aHSq1jz_AfYnJmLfygEflgUrQpqQEPnY,20157
 dapla_metadata/standards/__init__.py,sha256=n8jnMrudLuScSdfQ4UMJorc-Ptg3Y1-ilT8zAaQnM70,179
 dapla_metadata/standards/name_validator.py,sha256=6-DQE_EKVd6UjL--EXpFcZDQtusVbSFaWaUY-CfOV2c,9184
 dapla_metadata/standards/standard_validators.py,sha256=tcCiCI76wUVtMzXA2oCgdauZc0uGgUi11FKu-t7KGwQ,3767
@@ -90,7 +91,7 @@ dapla_metadata/variable_definitions/_utils/constants.py,sha256=zr5FNVCEz6TM9PVEr
 dapla_metadata/variable_definitions/_utils/files.py,sha256=JbPgPNQ7iA38juMqGEdcg5OjZZUwCb6NQtPL0AEspD0,10933
 dapla_metadata/variable_definitions/_utils/template_files.py,sha256=7fcc7yEHOl5JUZ698kqj4IiikXPHBi3SrAVOk4wqQtw,3308
 dapla_metadata/variable_definitions/_utils/variable_definition_files.py,sha256=sGhcSpckR9NtYGNh2oVkiCd5SI3bbJEBhc1PA2uShs0,4701
-dapla_toolbelt_metadata-0.9.4.dist-info/METADATA,sha256=xJyJK0IwI1cbzZyCNPNnBnGEAjHcwcwtjMPMt_MysrM,4723
-dapla_toolbelt_metadata-0.9.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-dapla_toolbelt_metadata-0.9.4.dist-info/licenses/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
-dapla_toolbelt_metadata-0.9.4.dist-info/RECORD,,
+dapla_toolbelt_metadata-0.9.5.dist-info/METADATA,sha256=tS5t2BEedPZXNfVIp1fkBPF6hCEub3w2G8nv8UsDoY4,4723
+dapla_toolbelt_metadata-0.9.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dapla_toolbelt_metadata-0.9.5.dist-info/licenses/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
+dapla_toolbelt_metadata-0.9.5.dist-info/RECORD,,