dapla-toolbelt-metadata 0.9.6.dev1759398171__tar.gz → 0.9.8__tar.gz

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/.github/workflows/release.yml

@@ -1,14 +1,11 @@
 name: Release
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
       - master
-      - release/0.9.6
-    tags:
-      - 'v*'
-
 
 jobs:
   release:
@@ -58,6 +55,7 @@ jobs:
           uv build
 
       - name: Publish package on PyPI
+        if: steps.check-version.outputs.tag
         uses: pypa/gh-action-pypi-publish@v1.13.0
 
       - name: Publish package on TestPyPI

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/.github/workflows/tests.yml

@@ -141,4 +141,4 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         # No need to run SonarCloud analysis if dependabot update or token not defined
         if: env.SONAR_TOKEN != '' && (github.actor != 'dependabot[bot]')
-        uses: SonarSource/sonarqube-scan-action@v5.3.1
+        uses: SonarSource/sonarqube-scan-action@v6.0.0

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dapla-toolbelt-metadata
-Version: 0.9.6.dev1759398171
+Version: 0.9.8
 Summary: Dapla Toolbelt Metadata
 Project-URL: homepage, https://github.com/statisticsnorway/dapla-toolbelt-metadata
 Project-URL: repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata
@@ -16,15 +16,19 @@ Requires-Dist: beautifulsoup4>=4.12.3
 Requires-Dist: cloudpathlib[gs]>=0.17.0
 Requires-Dist: google-auth>=2.38.0
 Requires-Dist: lxml>=5.3.1
+Requires-Dist: pandas>=2.3.3
 Requires-Dist: pyarrow>=8.0.0
 Requires-Dist: pydantic>=2.5.2
 Requires-Dist: pyjwt>=2.8.0
+Requires-Dist: python-dateutil>=2.9.0.post0
 Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: pytz>=2025.2
 Requires-Dist: requests>=2.31.0
 Requires-Dist: ruamel-yaml>=0.18.10
 Requires-Dist: ssb-datadoc-model<9.0.0,>=8.0.0
 Requires-Dist: ssb-klass-python>=1.0.1
 Requires-Dist: typing-extensions>=4.12.2
+Requires-Dist: urllib3>=2.5.0
 Description-Content-Type: text/markdown
 
 # Dapla Toolbelt Metadata

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/noxfile.py

@@ -31,6 +31,7 @@ TESTS_SESSIONS_DEPENDENCIES = {
     "requests-mock",
     "faker",
     "testcontainers[generic]",
+    "typeguard"
 }
 
 
@@ -152,6 +153,7 @@ def mypy(session: nox.Session) -> None:
         "types-beautifulsoup4",
         "types-docker",
         "faker",
+        "typeguard"
     )
     session.run("mypy", *args)
     if not session.posargs:

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "dapla-toolbelt-metadata"
 description = "Dapla Toolbelt Metadata"
 license = "MIT"
-version = "0.9.6.dev.1759398171"
+version = "0.9.8"
 classifiers = ["Development Status :: 4 - Beta"]
 readme = "README.md"
 authors = [{ name = "Statistics Norway", email = "metadata@ssb.no" }]
@@ -22,6 +22,10 @@ dependencies = [
     'ruamel-yaml >=0.18.10',
     'google-auth >=2.38.0',
     'lxml >=5.3.1',
+    "pandas>=2.3.3",
+    "urllib3>=2.5.0",
+    "pytz>=2025.2",
+    "python-dateutil>=2.9.0.post0",
 ]
 
 [project.urls]

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/src/dapla_metadata/_shared/config.py

@@ -13,9 +13,6 @@ from dotenv import load_dotenv
 from dapla_metadata._shared.enums import DaplaEnvironment
 from dapla_metadata._shared.enums import DaplaRegion
 from dapla_metadata._shared.enums import DaplaService
-from dapla_metadata.datasets.utility.constants import (
-    DATADOC_STATISTICAL_SUBJECT_SOURCE_URL,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -28,6 +25,12 @@ DAPLA_SERVICE = "DAPLA_SERVICE"
 DAPLA_GROUP_CONTEXT = "DAPLA_GROUP_CONTEXT"
 OIDC_TOKEN = "OIDC_TOKEN"  # noqa: S105
 
+
+DATADOC_STATISTICAL_SUBJECT_SOURCE_URL_DEFAULT = (
+    "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
+)
+
+
 env_loaded = False
 
 
@@ -73,7 +76,7 @@ def get_statistical_subject_source_url() -> str | None:
     """Get the URL to the statistical subject source."""
     return (
         get_config_item("DATADOC_STATISTICAL_SUBJECT_SOURCE_URL")
-        or DATADOC_STATISTICAL_SUBJECT_SOURCE_URL
+        or DATADOC_STATISTICAL_SUBJECT_SOURCE_URL_DEFAULT
     )
 
 

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/src/dapla_metadata/datasets/core.py

@@ -33,7 +33,12 @@ from dapla_metadata.datasets.utility.constants import (
 from dapla_metadata.datasets.utility.constants import METADATA_DOCUMENT_FILE_SUFFIX
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
+from dapla_metadata.datasets.utility.urn import convert_uris_to_urns
+from dapla_metadata.datasets.utility.urn import klass_urn_converter
+from dapla_metadata.datasets.utility.urn import vardef_urn_converter
 from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
+from dapla_metadata.datasets.utility.utils import VariableListType
+from dapla_metadata.datasets.utility.utils import VariableType
 from dapla_metadata.datasets.utility.utils import calculate_percentage
 from dapla_metadata.datasets.utility.utils import derive_assessment_from_state
 from dapla_metadata.datasets.utility.utils import get_timestamp_now
@@ -108,8 +113,8 @@ class Datadoc:
         self.container: all_optional_model.MetadataContainer | None = None
         self.dataset_path: pathlib.Path | CloudPath | None = None
         self.dataset = all_optional_model.Dataset()
-        self.variables: list = []
-        self.variables_lookup: dict[str, all_optional_model.Variable] = {}
+        self.variables: VariableListType = []
+        self.variables_lookup: dict[str, VariableType] = {}
         self.explicitly_defined_metadata_document = False
         self.dataset_consistency_status: list[DatasetConsistencyStatus] = []
         if metadata_document_path:
@@ -204,22 +209,24 @@ class Datadoc:
         else:
             self._set_metadata(existing_metadata or extracted_metadata)
 
-        set_default_values_variables(self.variables)
-        set_default_values_dataset(cast("all_optional_model.Dataset", self.dataset))
-        set_dataset_owner(self.dataset)
-        self._create_variables_lookup()
-
     def _set_metadata(
         self,
-        merged_metadata: OptionalDatadocMetadataType,
+        metadata: OptionalDatadocMetadataType,
     ) -> None:
-        if not merged_metadata or not (
-            merged_metadata.dataset and merged_metadata.variables
-        ):
+        if not metadata or not (metadata.dataset and metadata.variables):
             msg = "Could not read metadata"
             raise ValueError(msg)
-        self.dataset = cast("all_optional_model.Dataset", merged_metadata.dataset)
-        self.variables = merged_metadata.variables
+        self.dataset = cast("all_optional_model.Dataset", metadata.dataset)
+        self.variables = metadata.variables
+
+        set_default_values_variables(self.variables)
+        set_default_values_dataset(cast("all_optional_model.Dataset", self.dataset))
+        set_dataset_owner(self.dataset)
+        convert_uris_to_urns(self.variables, "definition_uri", [vardef_urn_converter])
+        convert_uris_to_urns(
+            self.variables, "classification_uri", [klass_urn_converter]
+        )
+        self._create_variables_lookup()
 
     def _create_variables_lookup(self) -> None:
         self.variables_lookup = {

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/src/dapla_metadata/datasets/utility/constants.py

@@ -94,10 +94,6 @@ DATASET_FIELDS_FROM_EXISTING_METADATA = [
 
 METADATA_DOCUMENT_FILE_SUFFIX = "__DOC.json"
 
-DATADOC_STATISTICAL_SUBJECT_SOURCE_URL = (
-    "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
-)
-
 PAPIS_STABLE_IDENTIFIER_TYPE = "FREG_SNR"
 PAPIS_ENCRYPTION_KEY_REFERENCE = "papis-common-key-1"
 DAEAD_ENCRYPTION_KEY_REFERENCE = "ssb-common-key-1"

dapla_toolbelt_metadata-0.9.8/src/dapla_metadata/datasets/utility/urn.py

@@ -0,0 +1,234 @@
+"""Validate, parse and render URNs."""
+
+import logging
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+from enum import Enum
+from enum import auto
+from typing import Literal
+
+from pydantic import AnyUrl
+
+from dapla_metadata._shared.config import get_dapla_environment
+from dapla_metadata._shared.enums import DaplaEnvironment
+from dapla_metadata.datasets.utility.utils import VariableListType
+
+logger = logging.getLogger(__name__)
+
+URN_ERROR_MESSAGE_BASE = "The URL is not in a supported format"
+
+URN_ERROR_MESSAGE_TEMPLATE = (
+    URN_ERROR_MESSAGE_BASE
+    + " for field '{field_name}' of variable '{short_name}'. URL: '{value}'. Please contact Team Metadata if this URL should be supported."
+)
+
+
+VARDEF_URL_TEMPLATE = "https://{subdomain}.{domain}/variable-definitions"
+
+
+class SsbNaisDomains(str, Enum):
+    """The available domains on SSBs Nais instance."""
+
+    TEST_EXTERNAL = "test.ssb.no"
+    TEST_INTERNAL = "intern.test.ssb.no"
+    PROD_EXTERNAL = "ssb.no"
+    PROD_INTERNAL = "intern.ssb.no"
+
+
+class ReferenceUrlTypes(Enum):
+    """The general category of the URL.
+
+    This can be useful to refer to when constructing a URL from a URN for a
+    specific context.
+    """
+
+    API = auto()
+    FRONTEND = auto()
+
+
+UrlVisibility = Literal["public", "internal"]
+
+
+@dataclass
+class UrnConverter:
+    """Converts URLs to URNs and vice versa.
+
+    Fields:
+        urn_base: The format for the URN, up to the identifier.
+        id_pattern: A capturing group pattern which matches identifiers for this resource.
+        url_bases: The list of all the different URL representations for a resource. There
+            will typically be a number of URL representations for a particular resource,
+            depending on which system or technology they are accessed through and other
+            technical factors. This list defines which concrete URLs can be considered
+            equivalent to a URN.
+    """
+
+    urn_base: str
+    id_pattern: str
+    url_bases: list[tuple[ReferenceUrlTypes, str]]
+
+    def _extract_id(self, url: str, pattern: re.Pattern[str]) -> str | None:
+        if match := pattern.match(url):
+            return match.group(1)
+        return None
+
+    def _build_pattern(self, url_base: str) -> re.Pattern[str]:
+        return re.compile(f"^{url_base}/{self.id_pattern}")
+
+    def get_urn(self, identifier: str) -> str:
+        """Build a URN for the given identifier."""
+        return f"{self.urn_base}:{identifier}"
+
+    def get_url(
+        self,
+        identifier: str,
+        url_type: ReferenceUrlTypes,
+        visibility: Literal["public", "internal"] = "public",
+    ) -> str | None:
+        """Build concrete URL to reference a resource.
+
+        There are typically multiple URLs used to refer to one resource, this method attempts to support known variations.
+
+        Args:
+            identifier (str): The identifier of the resource the URL refers to.
+            url_type (ReferenceUrlTypes): The representation type of the URL
+            visibility (UrlVisibility, optional): Whether the URL should be that which is publicly available or not. Defaults to "public".
+
+        Returns:
+            str | None: The concrete URL. None if we cannot satisfy the supplied requirements.
+        """
+        candidates = [base[-1] for base in self.url_bases if base[0] == url_type]
+
+        def matches_visibility(url: str, visibility: UrlVisibility):
+            return (".intern." in url) is (visibility == "internal")
+
+        def matches_environment(url: str):
+            current_environment = get_dapla_environment()
+            if current_environment == DaplaEnvironment.TEST:
+                return ".test." in url
+            return ".test." not in url
+
+        if url := next(
+            (
+                url
+                for url in candidates
+                if matches_visibility(url, visibility) and matches_environment(url)
+            ),
+            None,
+        ):
+            return url + "/" + identifier
+        return None
+
+    def get_id(self, urn_or_url: str | AnyUrl) -> str | None:
+        """Get an identifier from a URN or URL.
+
+        Args:
+            urn_or_url (str | AnyUrl): The URN or URL refering to a particular resource
+
+        Returns:
+            str | None: The identifier for the resource, or None if it cannot be extracted.
+        """
+        if str(urn_or_url).startswith(self.urn_base):
+            return str(urn_or_url).removeprefix(self.urn_base + ":")
+        return self._extract_id_from_url(urn_or_url)
+
+    def is_id(self, value: str) -> bool:
+        """Check if the value is an identifier for this URN type.
+
+        Args:
+            value (str): The value to check.
+        """
+        if not isinstance(value, str):
+            # Mypy thinks it's impossible to reach this branch, but there are no guarantees in Python.
+            return False  # type: ignore [unreachable]
+        pattern = re.compile(f"^{self.id_pattern}$")
+        return bool(pattern.match(value))
+
+    def _extract_id_from_url(self, url: str | AnyUrl) -> str | None:
+        patterns = (self._build_pattern(url[-1]) for url in self.url_bases)
+        matches = (self._extract_id(str(url), p) for p in patterns)
+        return next((m for m in matches if m), None)
+
+    def convert_url_to_urn(self, url: str | AnyUrl) -> AnyUrl | None:
+        """Convert a URL to a generalized URN for that same resource.
+
+        Args:
+            url (str | AnyUrl): The URL to convert.
+
+        Returns:
+            str | None: The URN or None if it can't be converted.
+        """
+        if str(url).startswith(self.urn_base):
+            # In this case the value is already in the expected format and nothing needs to be done.
+            return AnyUrl(url)
+        if identifier := self._extract_id_from_url(url):
+            return AnyUrl(self.get_urn(identifier))
+
+        return None
+
+
+vardef_urn_converter = UrnConverter(
+    urn_base="urn:ssb:variable-definition:vardef",
+    id_pattern=r"([a-z0-9]{8})",
+    url_bases=[
+        *[
+            (
+                ReferenceUrlTypes.API,
+                VARDEF_URL_TEMPLATE.format(
+                    subdomain="metadata", domain=nais_domain.value
+                ),
+            )
+            for nais_domain in SsbNaisDomains
+        ],
+        *[
+            (
+                ReferenceUrlTypes.FRONTEND,
+                VARDEF_URL_TEMPLATE.format(
+                    subdomain="catalog", domain=nais_domain.value
+                ),
+            )
+            for nais_domain in SsbNaisDomains
+        ],
+    ],
+)
+
+klass_urn_converter = UrnConverter(
+    urn_base="urn:ssb:classification:klass",
+    id_pattern=r"([0-9]{1,5})",
+    url_bases=[
+        (ReferenceUrlTypes.FRONTEND, "https://www.ssb.no/klass/klassifikasjoner"),
+        (ReferenceUrlTypes.FRONTEND, "https://www.ssb.no/en/klass/klassifikasjoner"),
+        (ReferenceUrlTypes.API, "https://data.ssb.no/api/klass/v1/classifications"),
+    ],
+)
+
+
+def convert_uris_to_urns(
+    variables: VariableListType, field_name: str, converters: Iterable[UrnConverter]
+) -> None:
+    """Where URIs are recognized URLs, convert them to URNs.
+
+    Where the value is not a known URL we preserve the value as it is and log an
+    ERROR level message.
+
+    Args:
+        variables (VariableListType): The list of variables.
+        field_name (str): The name of the field which has URLs to convert to URNs
+        converters (Iterable[UrnConverter]): One or more converters which implement
+            conversion of URLs into one specific URN format. These will typically be
+            specific to an individual metadata reference system.
+    """
+    for v in variables:
+        field = getattr(v, field_name, None)
+        if field:
+            if urn := next((c.convert_url_to_urn(field) for c in converters), None):
+                setattr(v, field_name, urn)
+            else:
+                logger.error(
+                    URN_ERROR_MESSAGE_TEMPLATE.format(
+                        field_name=field_name,
+                        short_name=v.short_name,
+                        value=field,
+                    )
+                )

{dapla_toolbelt_metadata-0.9.6.dev1759398171 → dapla_toolbelt_metadata-0.9.8}/src/dapla_metadata/datasets/utility/utils.py

@@ -121,7 +121,7 @@ def derive_assessment_from_state(state: DataSetState) -> Assessment:
             return Assessment.SENSITIVE
 
 
-def set_default_values_variables(variables: list) -> None:
+def set_default_values_variables(variables: VariableListType) -> None:
     """Set default values on variables.
 
     Args: