dapla-toolbelt-metadata 0.9.7__tar.gz → 0.9.9__tar.gz

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dapla-toolbelt-metadata
-Version: 0.9.7
+Version: 0.9.9
 Summary: Dapla Toolbelt Metadata
 Project-URL: homepage, https://github.com/statisticsnorway/dapla-toolbelt-metadata
 Project-URL: repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/noxfile.py

@@ -31,6 +31,7 @@ TESTS_SESSIONS_DEPENDENCIES = {
     "requests-mock",
     "faker",
     "testcontainers[generic]",
+    "typeguard"
 }
 
 
@@ -152,6 +153,7 @@ def mypy(session: nox.Session) -> None:
         "types-beautifulsoup4",
         "types-docker",
         "faker",
+        "typeguard"
     )
     session.run("mypy", *args)
     if not session.posargs:

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "dapla-toolbelt-metadata"
 description = "Dapla Toolbelt Metadata"
 license = "MIT"
-version = "0.9.7"
+version = "0.9.9"
 classifiers = ["Development Status :: 4 - Beta"]
 readme = "README.md"
 authors = [{ name = "Statistics Norway", email = "metadata@ssb.no" }]

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/src/dapla_metadata/_shared/config.py

@@ -13,9 +13,6 @@ from dotenv import load_dotenv
 from dapla_metadata._shared.enums import DaplaEnvironment
 from dapla_metadata._shared.enums import DaplaRegion
 from dapla_metadata._shared.enums import DaplaService
-from dapla_metadata.datasets.utility.constants import (
-    DATADOC_STATISTICAL_SUBJECT_SOURCE_URL,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -28,6 +25,12 @@ DAPLA_SERVICE = "DAPLA_SERVICE"
 DAPLA_GROUP_CONTEXT = "DAPLA_GROUP_CONTEXT"
 OIDC_TOKEN = "OIDC_TOKEN"  # noqa: S105
 
+
+DATADOC_STATISTICAL_SUBJECT_SOURCE_URL_DEFAULT = (
+    "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
+)
+
+
 env_loaded = False
 
 
@@ -73,7 +76,7 @@ def get_statistical_subject_source_url() -> str | None:
     """Get the URL to the statistical subject source."""
     return (
         get_config_item("DATADOC_STATISTICAL_SUBJECT_SOURCE_URL")
-        or DATADOC_STATISTICAL_SUBJECT_SOURCE_URL
+        or DATADOC_STATISTICAL_SUBJECT_SOURCE_URL_DEFAULT
     )
 
 

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/src/dapla_metadata/datasets/_merge.py

@@ -78,7 +78,7 @@ class DatasetConsistencyStatus:
 
 def check_dataset_consistency(
     new_dataset_path: Path | CloudPath,
-    existing_dataset_path: Path,
+    existing_dataset_path: Path | CloudPath,
 ) -> list[DatasetConsistencyStatus]:
     """Run consistency tests.
 

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/src/dapla_metadata/datasets/core.py

@@ -6,7 +6,6 @@ import copy
 import json
 import logging
 from concurrent.futures import ThreadPoolExecutor
-from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import cast
 
@@ -175,7 +174,7 @@ class Datadoc:
         ):
             self.dataset_consistency_status = check_dataset_consistency(
                 self.dataset_path,
-                Path(self.metadata_document),
+                self.metadata_document,
             )
             self.dataset_consistency_status.extend(
                 check_variables_consistency(

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/src/dapla_metadata/datasets/utility/constants.py

@@ -94,10 +94,6 @@ DATASET_FIELDS_FROM_EXISTING_METADATA = [
 
 METADATA_DOCUMENT_FILE_SUFFIX = "__DOC.json"
 
-DATADOC_STATISTICAL_SUBJECT_SOURCE_URL = (
-    "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
-)
-
 PAPIS_STABLE_IDENTIFIER_TYPE = "FREG_SNR"
 PAPIS_ENCRYPTION_KEY_REFERENCE = "papis-common-key-1"
 DAEAD_ENCRYPTION_KEY_REFERENCE = "ssb-common-key-1"

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/src/dapla_metadata/datasets/utility/urn.py

@@ -6,9 +6,12 @@ from collections.abc import Iterable
 from dataclasses import dataclass
 from enum import Enum
 from enum import auto
+from typing import Literal
 
 from pydantic import AnyUrl
 
+from dapla_metadata._shared.config import get_dapla_environment
+from dapla_metadata._shared.enums import DaplaEnvironment
 from dapla_metadata.datasets.utility.utils import VariableListType
 
 logger = logging.getLogger(__name__)
@@ -44,6 +47,9 @@ class ReferenceUrlTypes(Enum):
     FRONTEND = auto()
 
 
+UrlVisibility = Literal["public", "internal"]
+
+
 @dataclass
 class UrnConverter:
     """Converts URLs to URNs and vice versa.
@@ -70,11 +76,81 @@ class UrnConverter:
     def _build_pattern(self, url_base: str) -> re.Pattern[str]:
         return re.compile(f"^{url_base}/{self.id_pattern}")
 
-    def build_urn(self, identifier: str) -> str:
+    def get_urn(self, identifier: str) -> str:
         """Build a URN for the given identifier."""
         return f"{self.urn_base}:{identifier}"
 
-    def convert_to_urn(self, url: str | AnyUrl) -> AnyUrl | None:
+    def get_url(
+        self,
+        identifier: str,
+        url_type: ReferenceUrlTypes,
+        visibility: Literal["public", "internal"] = "public",
+    ) -> str | None:
+        """Build concrete URL to reference a resource.
+
+        There are typically multiple URLs used to refer to one resource, this method attempts to support known variations.
+
+        Args:
+            identifier (str): The identifier of the resource the URL refers to.
+            url_type (ReferenceUrlTypes): The representation type of the URL
+            visibility (UrlVisibility, optional): Whether the URL should be that which is publicly available or not. Defaults to "public".
+
+        Returns:
+            str | None: The concrete URL. None if we cannot satisfy the supplied requirements.
+        """
+        candidates = [base[-1] for base in self.url_bases if base[0] == url_type]
+
+        def matches_visibility(url: str, visibility: UrlVisibility):
+            return (".intern." in url) is (visibility == "internal")
+
+        def matches_environment(url: str):
+            current_environment = get_dapla_environment()
+            if current_environment == DaplaEnvironment.TEST:
+                return ".test." in url
+            return ".test." not in url
+
+        if url := next(
+            (
+                url
+                for url in candidates
+                if matches_visibility(url, visibility) and matches_environment(url)
+            ),
+            None,
+        ):
+            return url + "/" + identifier
+        return None
+
+    def get_id(self, urn_or_url: str | AnyUrl) -> str | None:
+        """Get an identifier from a URN or URL.
+
+        Args:
+            urn_or_url (str | AnyUrl): The URN or URL refering to a particular resource
+
+        Returns:
+            str | None: The identifier for the resource, or None if it cannot be extracted.
+        """
+        if str(urn_or_url).startswith(self.urn_base):
+            return str(urn_or_url).removeprefix(self.urn_base + ":")
+        return self._extract_id_from_url(urn_or_url)
+
+    def is_id(self, value: str) -> bool:
+        """Check if the value is an identifier for this URN type.
+
+        Args:
+            value (str): The value to check.
+        """
+        if not isinstance(value, str):
+            # Mypy thinks it's impossible to reach this branch, but there are no guarantees in Python.
+            return False  # type: ignore [unreachable]
+        pattern = re.compile(f"^{self.id_pattern}$")
+        return bool(pattern.match(value))
+
+    def _extract_id_from_url(self, url: str | AnyUrl) -> str | None:
+        patterns = (self._build_pattern(url[-1]) for url in self.url_bases)
+        matches = (self._extract_id(str(url), p) for p in patterns)
+        return next((m for m in matches if m), None)
+
+    def convert_url_to_urn(self, url: str | AnyUrl) -> AnyUrl | None:
         """Convert a URL to a generalized URN for that same resource.
 
         Args:
@@ -86,11 +162,8 @@ class UrnConverter:
         if str(url).startswith(self.urn_base):
             # In this case the value is already in the expected format and nothing needs to be done.
             return AnyUrl(url)
-        patterns = (self._build_pattern(url[-1]) for url in self.url_bases)
-        matches = (self._extract_id(str(url), p) for p in patterns)
-        identifier = next((m for m in matches if m), None)
-        if identifier:
-            return AnyUrl(self.build_urn(identifier))
+        if identifier := self._extract_id_from_url(url):
+            return AnyUrl(self.get_urn(identifier))
 
         return None
 
@@ -149,7 +222,7 @@ def convert_uris_to_urns(
     for v in variables:
         field = getattr(v, field_name, None)
         if field:
-            if urn := next((c.convert_to_urn(field) for c in converters), None):
+            if urn := next((c.convert_url_to_urn(field) for c in converters), None):
                 setattr(v, field_name, urn)
             else:
                 logger.error(

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/src/dapla_metadata/variable_definitions/vardok_id.py

@@ -33,7 +33,11 @@ class VardokId(VardokIdResponse):
         model: VardokIdResponse,
     ) -> "VardokId":
         """Create a VariableDefinition instance from a CompleteResponse."""
-        return VardokId.model_construct(**model.model_dump())
+        self = VardokId.from_dict(model.model_dump())
+        if not self:
+            msg = f"Could not construct a VardokId instance from {model}"
+            raise ValueError(msg)
+        return self
 
     def __str__(self) -> str:
         """Format as indented YAML."""

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/src/dapla_metadata/variable_definitions/vardok_vardef_id_pair.py

@@ -32,7 +32,11 @@ class VardokVardefIdPair(VardokVardefIdPairResponse):
         model: VardokVardefIdPairResponse,
     ) -> "VardokVardefIdPair":
         """Create a VardokVardefIdPair instance from a VardokVardefIdPairResponse."""
-        return VardokVardefIdPair.model_construct(**model.model_dump())
+        self = VardokVardefIdPair.from_dict(model.model_dump())
+        if not self:
+            msg = f"Could not construct a VardokVardefIdPair instance from {model}"
+            raise ValueError(msg)
+        return self
 
     def __str__(self) -> str:
         """Format as indented YAML."""

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/src/dapla_metadata/variable_definitions/variable_definition.py

@@ -82,7 +82,11 @@ class VariableDefinition(CompleteResponse):
         model: CompleteResponse,
     ) -> "VariableDefinition":
         """Create a VariableDefinition instance from a CompleteResponse."""
-        return VariableDefinition.model_construct(**model.model_dump())
+        self = VariableDefinition.from_dict(model.model_dump())
+        if not self:
+            msg = f"Could not construct a VariableDefinition instance from {model}"
+            raise ValueError(msg)
+        return self
 
     @vardef_exception_handler
     def list_validity_periods(self) -> list["VariableDefinition"]:

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/tests/datasets/test_dataset_consistency.py

@@ -2,6 +2,8 @@ import contextlib
 from pathlib import Path
 
 import pytest
+from cloudpathlib.local import LocalGSClient
+from cloudpathlib.local import LocalGSPath
 from datadoc_model.all_optional.model import DatadocMetadata
 from datadoc_model.all_optional.model import DataType
 from datadoc_model.all_optional.model import Variable
@@ -21,6 +23,7 @@ from dapla_metadata.datasets._merge import InconsistentDatasetsWarning
 from dapla_metadata.datasets._merge import check_dataset_consistency
 from dapla_metadata.datasets._merge import check_ready_to_merge
 from dapla_metadata.datasets._merge import check_variables_consistency
+from dapla_metadata.datasets.core import Datadoc
 from tests.datasets.constants import TEST_BUCKET_NAMING_STANDARD_COMPATIBLE_PATH
 from tests.datasets.constants import VARIABLE_DATA_TYPES
 from tests.datasets.constants import VARIABLE_SHORT_NAMES
@@ -227,3 +230,61 @@ def test_check_variables_consistency(
                 assert r.success, f"'{r.message}' failed but should have passed"
     else:
         assert all(r.success for r in results)
+
+
+def test_bucket_check_ok_when_both_params_are_gs_paths(mocker):
+    """Test the functionality of `check_dataset_consistency`.
+
+    Verify the desired behavior when BOTH parameters are gs-paths (no Path() conversion occurs). In
+    this case no `Bucket name` warning should be issued.
+    """
+    _patch_gs_utils(mocker)
+    resource_json = (
+        Path(__file__).parent
+        / "resources"
+        / "existing_metadata_file"
+        / "person_testdata_p2020-12-31_p2020-12-31_v1__DOC.json"
+    )
+    resource_parquet = (
+        Path(__file__).parent
+        / "resources"
+        / "datasets"
+        / "person_testdata_p2021-12-31_p2021-12-31_v1.parquet"
+    )
+    assert resource_parquet.exists(), f"Parquet resource missing: {resource_parquet}"
+    assert resource_json.exists(), f"JSON resource missing: {resource_json}"
+    directory = "ssb-dapla-felles-data-produkt-test/datadoc/brukertest/10/sykefratot/klargjorte_data/"
+    dataset_gs = LocalGSPath(
+        f"gs://{directory}person_testdata_p2021-12-31_p2021-12-31_v1.parquet"
+    )
+    metadata_doc_gs = LocalGSPath(
+        f"gs://{directory}person_testdata_p2021-12-31_p2021-12-31_v1__DOC.json"
+    )
+    metadata_doc_gs.parent.mkdir(parents=True, exist_ok=True)
+    metadata_doc_gs.write_text(resource_json.read_text(), encoding="utf-8")
+    assert metadata_doc_gs.exists(), "Metadata document was not written to LocalGS"
+    dataset_gs.parent.mkdir(parents=True, exist_ok=True)
+    dataset_gs.write_bytes(resource_parquet.read_bytes())
+    assert dataset_gs.exists(), "Parquet file was not written to LocalGS"
+    dd = Datadoc(
+        dataset_path=str(dataset_gs),
+        metadata_document_path=str(metadata_doc_gs),
+        errors_as_warnings=True,
+    )
+    _assert_bucket_ok(dd.dataset_consistency_status, BUCKET_NAME_MESSAGE)
+
+
+def _assert_bucket_ok(results: list, expected_msg: str):
+    bucket_checks = [r for r in results if expected_msg in r.message]
+    assert bucket_checks, f"Expected at least one check for {expected_msg}."
+    assert all(r.success for r in bucket_checks), (
+        f"Expected {expected_msg} to have success == True when both are gs-paths."
+    )
+
+
+def _patch_gs_utils(mocker):
+    mocker.patch(
+        "dapla_metadata.datasets.utility.utils.google.auth.default", autospec=True
+    )
+    mocker.patch("dapla_metadata.datasets.utility.utils.GSClient", LocalGSClient)
+    mocker.patch("dapla_metadata.datasets.utility.utils.GSPath", LocalGSPath)

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/tests/datasets/test_urn.py

@@ -3,17 +3,20 @@ import logging
 import datadoc_model.all_optional.model as all_optional_model
 import pytest
 from pydantic import AnyUrl
+from typeguard import suppress_type_checks
 
 from dapla_metadata.datasets.core import Datadoc
 from dapla_metadata.datasets.utility.urn import URN_ERROR_MESSAGE_BASE
+from dapla_metadata.datasets.utility.urn import ReferenceUrlTypes
 from dapla_metadata.datasets.utility.urn import SsbNaisDomains
+from dapla_metadata.datasets.utility.urn import UrlVisibility
 from dapla_metadata.datasets.utility.urn import klass_urn_converter
 from dapla_metadata.datasets.utility.urn import vardef_urn_converter
 
 EXAMPLE_KLASS_ID = "91"
-EXAMPLE_KLASS_URN = AnyUrl(klass_urn_converter.build_urn(EXAMPLE_KLASS_ID))
+EXAMPLE_KLASS_URN = AnyUrl(klass_urn_converter.get_urn(EXAMPLE_KLASS_ID))
 EXAMPLE_VARDEF_ID = "hd8sks89"
-EXAMPLE_VARDEF_URN = AnyUrl(vardef_urn_converter.build_urn(EXAMPLE_VARDEF_ID))
+EXAMPLE_VARDEF_URN = AnyUrl(vardef_urn_converter.get_urn(EXAMPLE_VARDEF_ID))
 
 VARIABLE_DEFINITION_URN_TEST_CASES = [
     *[
@@ -52,7 +55,7 @@ def test_convert_to_vardef_urn(
     expected_result: AnyUrl | None,
     expect_warning: bool,  # noqa: ARG001
 ):
-    assert vardef_urn_converter.convert_to_urn(case) == expected_result
+    assert vardef_urn_converter.convert_url_to_urn(case) == expected_result
 
 
 @pytest.mark.parametrize(
@@ -88,6 +91,66 @@ def test_convert_to_vardef_urn_end_to_end(
     assert (URN_ERROR_MESSAGE_BASE in caplog.text) is expect_warning
 
 
+@pytest.mark.parametrize("identifier", [EXAMPLE_VARDEF_ID])
+@pytest.mark.parametrize("visibility", ["internal", "public"])
+@pytest.mark.parametrize(
+    "url_type", [ReferenceUrlTypes.API, ReferenceUrlTypes.FRONTEND]
+)
+def test_vardef_get_url(
+    url_type: ReferenceUrlTypes, visibility: UrlVisibility, identifier: str
+):
+    url = vardef_urn_converter.get_url(identifier, url_type, visibility)
+    assert url is not None
+    assert url.endswith("/" + identifier)
+    if url_type == ReferenceUrlTypes.API:
+        assert url.startswith("https://metadata.")
+    else:
+        assert url.startswith("https://catalog.")
+    if visibility == "internal":
+        assert ".intern." in url
+    else:
+        assert ".intern." not in url
+
+
+@pytest.mark.parametrize(
+    ("urn_or_url", "identifier"),
+    [
+        ("https://www.vg.no", None),
+        (EXAMPLE_VARDEF_URN, EXAMPLE_VARDEF_ID),
+        (
+            f"https://catalog.ssb.no/variable-definitions/{EXAMPLE_VARDEF_ID}",
+            EXAMPLE_VARDEF_ID,
+        ),
+        (
+            AnyUrl(
+                f"https://metadata.intern.test.ssb.no/variable-definitions/{EXAMPLE_VARDEF_ID}"
+            ),
+            EXAMPLE_VARDEF_ID,
+        ),
+    ],
+)
+def test_vardef_get_id(urn_or_url: str | AnyUrl, identifier: str | None):
+    assert vardef_urn_converter.get_id(urn_or_url) == identifier
+
+
+@pytest.mark.parametrize(
+    ("identifier", "expected"),
+    [
+        (EXAMPLE_VARDEF_ID, True),
+        ("invalid-id", False),
+        ("123", False),
+        ("12345678", True),
+        ("123456789", False),
+        ("abcdEFGH", False),
+        ("", False),
+        (None, False),
+    ],
+)
+@suppress_type_checks
+def test_vardef_is_id(identifier: str, expected: bool):
+    assert vardef_urn_converter.is_id(identifier) is expected
+
+
 CLASSIFICATION_URN_TEST_CASES = [
     (
         f"https://www.ssb.no/klass/klassifikasjoner/{EXAMPLE_KLASS_ID}",
@@ -126,7 +189,7 @@ def test_convert_to_klass_urn(
     expected_result: str,
     expect_warning: bool,  # noqa: ARG001
 ):
-    assert klass_urn_converter.convert_to_urn(case) == expected_result
+    assert klass_urn_converter.convert_url_to_urn(case) == expected_result
 
 
 @pytest.mark.parametrize(
@@ -160,3 +223,63 @@ def test_convert_to_klass_urn_end_to_end(
     else:
         assert meta.variables[0].classification_uri == expected_result
     assert (URN_ERROR_MESSAGE_BASE in caplog.text) is expect_warning
+
+
+@pytest.mark.parametrize("identifier", [EXAMPLE_KLASS_ID])
+@pytest.mark.parametrize("visibility", ["internal", "public"])
+@pytest.mark.parametrize(
+    "url_type", [ReferenceUrlTypes.API, ReferenceUrlTypes.FRONTEND]
+)
+def test_klass_get_url(
+    url_type: ReferenceUrlTypes, visibility: UrlVisibility, identifier: str
+):
+    url = klass_urn_converter.get_url(identifier, url_type, visibility)
+    if visibility == "internal":
+        # Klass doesn't have an internal representation
+        assert url is None
+    else:
+        assert url is not None
+        assert url.endswith("/" + identifier)
+        if url_type == ReferenceUrlTypes.API:
+            assert url.startswith("https://data.")
+        else:
+            assert url.startswith("https://www.ssb.")
+
+
+@pytest.mark.parametrize(
+    ("urn_or_url", "identifier"),
+    [
+        ("https://www.vg.no", None),
+        (EXAMPLE_KLASS_URN, EXAMPLE_KLASS_ID),
+        (
+            f"https://www.ssb.no/klass/klassifikasjoner/{EXAMPLE_KLASS_ID}",
+            EXAMPLE_KLASS_ID,
+        ),
+        (
+            AnyUrl(
+                f"https://data.ssb.no/api/klass/v1/classifications/{EXAMPLE_KLASS_ID}"
+            ),
+            EXAMPLE_KLASS_ID,
+        ),
+    ],
+)
+def test_klass_get_id(urn_or_url: str | AnyUrl, identifier: str | None):
+    assert klass_urn_converter.get_id(urn_or_url) == identifier
+
+
+@pytest.mark.parametrize(
+    ("identifier", "expected"),
+    [
+        (EXAMPLE_KLASS_ID, True),
+        ("invalid-id", False),
+        ("123", True),
+        ("1", True),
+        ("123456789", False),
+        ("abcdEFGH", False),
+        ("", False),
+        (None, False),
+    ],
+)
+@suppress_type_checks
+def test_klass_is_id(identifier: str, expected: bool):
+    assert klass_urn_converter.is_id(identifier) is expected

{dapla_toolbelt_metadata-0.9.7 → dapla_toolbelt_metadata-0.9.9}/tests/variable_definitions/test_variable_definition.py

@@ -253,6 +253,30 @@ def test_create_patch_change_short_name(variable_definition: VariableDefinition)
     mock_create_patch.assert_not_called()
 
 
+def test_from_model(
+    variable_definition: VariableDefinition,
+):
+    """This covers a bug that was reported in.
+
+    It was caused by an incorrect implementation in the from_model
+    method. The only thing we're testing is that no exceptions are
+    raised in this flow.
+    """
+    mock_api_response = MagicMock(name="VariableDefinition")
+    mock_api_response.classification_reference = "702"
+    with patch.object(
+        VariableDefinition,
+        "create_patch",
+        return_value=mock_api_response,
+    ) as mock_create_patch:
+        my_patch = VariableDefinition.from_model(variable_definition)
+        my_patch.create_patch_from_file(
+            file_path=VARIABLE_DEFINITION_EDITING_FILES_DIR
+            / "variable_definition_landbak_wypvb3wd_2025-05-02T10-06-20.yaml",
+        )
+        mock_create_patch.assert_called_once()
+
+
 def test_create_patch_from_file_path_not_set(variable_definition: VariableDefinition):
     my_patch = variable_definition
     my_patch.set_file_path(file_path=None)