dapla-toolbelt-metadata 0.2.6__tar.gz → 0.4.0__tar.gz

{dapla_toolbelt_metadata-0.2.6 → dapla_toolbelt_metadata-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dapla-toolbelt-metadata
-Version: 0.2.6
+Version: 0.4.0
 Summary: Dapla Toolbelt Metadata
 Home-page: https://github.com/statisticsnorway/dapla-toolbelt-metadata
 License: MIT

{dapla_toolbelt_metadata-0.2.6 → dapla_toolbelt_metadata-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dapla-toolbelt-metadata"
-version = "0.2.6"
+version = "0.4.0"
 description = "Dapla Toolbelt Metadata"
 authors = ["Team Metadata <metadata@ssb.no>"]
 license = "MIT"
@@ -62,24 +62,35 @@ types-setuptools = "*"
 types-beautifulsoup4 = ">=4.12.0.20240511"
 ipykernel = "^6.29.5"
 rich = "^13.9.4"
+bpython = "^0.24"
+testcontainers = { version = "^4.8.2", extras = ["generic"] }
 
 [tool.pytest.ini_options]
 pythonpath = ["src"]
 
-
 [tool.coverage.paths]
 source = ["src", "*/site-packages"]
-tests = ["tests", "*/tests"]
+tests = ["tests", "*/tests", "*/test"]
 
 [tool.coverage.run]
 branch = true
 source = ["dapla_metadata"]
-omit = ["tests/*", "__init__.py"]
+omit = [
+    "tests/*",
+    "__init__.py",
+    "*/dapla_metadata/variable_definitions/generated/*",
+]
 relative_files = true
 
 [tool.coverage.report]
 show_missing = true
 fail_under = 80
+omit = ["*/dapla_metadata/variable_definitions/generated/*"]
+
+[tool.black]
+# We don't want to format generated files because it creates large diffs
+# when generated code is updated.
+extend-exclude = "/generated/"
 
 [tool.mypy]
 plugins = ["pydantic.mypy"]
@@ -88,10 +99,20 @@ warn_unreachable = true
 pretty = true
 show_column_numbers = true
 show_error_context = true
+# Generated code is not high quality, just ignore the problems there
+exclude = "generated/vardef_client/.*"
+
+[[tool.mypy.overrides]]
+module = ["dapla_metadata.variable_definitions.generated.vardef_client.*"]
+warn_unreachable = false
+ignore_errors = true
+disable_error_code = ["unreachable"]
+
 
 [[tool.mypy.overrides]]
 # Allow missing type hints in third-party libraries without type information.
 module = [
+    "nox",
     "dapla",
     "gcsfs",
     "pyarrow",
@@ -99,6 +120,9 @@ module = [
     "datadoc_model",
     "datadoc_model.model",
     "pytest_mock",
+    "testcontainers.*",
+    "yaml",
+    "httpx",
 ]
 ignore_missing_imports = true
 
@@ -149,6 +173,9 @@ max-complexity = 15
 [tool.ruff.lint.pydocstyle]
 convention = "google" # You can also use "numpy".
 
+[tool.ruff.lint.flake8-pytest-style]
+fixture-parentheses = false
+
 [tool.ruff.lint.pep8-naming]
 classmethod-decorators = [
     "classmethod",

{dapla_toolbelt_metadata-0.2.6 → dapla_toolbelt_metadata-0.4.0}/src/dapla_metadata/__init__.py

@@ -3,3 +3,4 @@
 import datadoc_model.model as datadoc_model
 
 from . import datasets
+from . import variable_definitions

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/_shared/__init__.py

@@ -0,0 +1 @@
+"""Utility code intended to be used by multiple packages within this project, but not by end users."""

{dapla_toolbelt_metadata-0.2.6/src/dapla_metadata/datasets → dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/_shared}/config.py

@@ -10,11 +10,12 @@ from pprint import pformat
 from dotenv import dotenv_values
 from dotenv import load_dotenv
 
+from dapla_metadata._shared.enums import DaplaEnvironment
+from dapla_metadata._shared.enums import DaplaRegion
+from dapla_metadata._shared.enums import DaplaService
 from dapla_metadata.datasets.utility.constants import (
     DATADOC_STATISTICAL_SUBJECT_SOURCE_URL,
 )
-from dapla_metadata.datasets.utility.enums import DaplaRegion
-from dapla_metadata.datasets.utility.enums import DaplaService
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +23,10 @@ DOT_ENV_FILE_PATH = Path(__file__).parent.joinpath(".env")
 
 JUPYTERHUB_USER = "JUPYTERHUB_USER"
 DAPLA_REGION = "DAPLA_REGION"
+DAPLA_ENVIRONMENT = "DAPLA_ENVIRONMENT"
 DAPLA_SERVICE = "DAPLA_SERVICE"
+DAPLA_GROUP_CONTEXT = "DAPLA_GROUP_CONTEXT"
+OIDC_TOKEN = "OIDC_TOKEN"  # noqa: S105
 
 env_loaded = False
 
@@ -38,43 +42,70 @@ def _load_dotenv_file() -> None:
         )
 
 
-def _get_config_item(item: str) -> str | None:
-    """Get a config item. Makes sure all access is logged."""
+def get_config_item(item: str, *, raising: bool = False) -> str | None:
+    """Get a config item. Makes sure all access is logged.
+
+    Args:
+        item: The name of the environment variable to obtain.
+        raising: `True` if an exception should be raised when the item isn't present.
+
+    Returns:
+        The set value or `None`
+
+    Raises:
+        OSError: Only if `raising` is True and the item is not found.
+    """
     _load_dotenv_file()
-    value = os.getenv(item)
+    value = os.environ.get(item)
+    if raising and not value:
+        msg = f"Environment variable {item} not defined."
+        raise OSError(msg)
     logger.debug("Config accessed. %s", item)
     return value
 
 
 def get_jupyterhub_user() -> str | None:
     """Get the JupyterHub user name."""
-    return _get_config_item(JUPYTERHUB_USER)
+    return get_config_item(JUPYTERHUB_USER)
 
 
 def get_statistical_subject_source_url() -> str | None:
     """Get the URL to the statistical subject source."""
     return (
-        _get_config_item("DATADOC_STATISTICAL_SUBJECT_SOURCE_URL")
+        get_config_item("DATADOC_STATISTICAL_SUBJECT_SOURCE_URL")
         or DATADOC_STATISTICAL_SUBJECT_SOURCE_URL
     )
 
 
 def get_dapla_region() -> DaplaRegion | None:
     """Get the Dapla region we're running on."""
-    if region := _get_config_item(DAPLA_REGION):
+    if region := get_config_item(DAPLA_REGION):
         return DaplaRegion(region)
 
     return None
 
 
+def get_dapla_environment() -> DaplaEnvironment | None:
+    """Get the Dapla environment we're running on."""
+    if env := get_config_item(DAPLA_ENVIRONMENT):
+        return DaplaEnvironment(env)
+
+    return None
+
+
 def get_dapla_service() -> DaplaService | None:
     """Get the Dapla service we're running on."""
-    if service := _get_config_item(DAPLA_SERVICE):
+    if service := get_config_item(DAPLA_SERVICE):
         return DaplaService(service)
 
     return None
 
 
-def get_oidc_token() -> str | None:
+def get_oidc_token(*, raising: bool = False) -> str | None:
     """Get the JWT token from the environment."""
-    return _get_config_item("OIDC_TOKEN")
+    return get_config_item(OIDC_TOKEN, raising=raising)
+
+
+def get_dapla_group_context(*, raising: bool = False) -> str | None:
+    """Get the group which the user has chosen to represent."""
+    return get_config_item(DAPLA_GROUP_CONTEXT, raising=raising)

{dapla_toolbelt_metadata-0.2.6/src/dapla_metadata/datasets/utility → dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/_shared}/enums.py

@@ -1,7 +1,3 @@
-"""Enumerations used in Datadoc."""
-
-from __future__ import annotations
-
 from enum import Enum
 
 
@@ -14,6 +10,14 @@ class DaplaRegion(str, Enum):
     CLOUD_RUN = "CLOUD_RUN"
 
 
+class DaplaEnvironment(str, Enum):
+    """Dapla lifecycle environment."""
+
+    PROD = "PROD"
+    TEST = "TEST"
+    DEV = "DEV"
+
+
 class DaplaService(str, Enum):
     """Dapla services."""
 
@@ -22,14 +26,3 @@ class DaplaService(str, Enum):
     VS_CODE = "VS_CODE"
     R_STUDIO = "R_STUDIO"
     KILDOMATEN = "KILDOMATEN"
-
-
-class SupportedLanguages(str, Enum):
-    """The list of languages metadata may be recorded in.
-
-    Reference: https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
-    """
-
-    NORSK_BOKMÅL = "nb"
-    NORSK_NYNORSK = "nn"
-    ENGLISH = "en"

{dapla_toolbelt_metadata-0.2.6/src/dapla_metadata/datasets → dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/_shared}/user_info.py

@@ -6,9 +6,9 @@ from typing import Protocol
 
 import jwt
 
-from dapla_metadata.datasets import config
-from dapla_metadata.datasets.utility.enums import DaplaRegion
-from dapla_metadata.datasets.utility.enums import DaplaService
+from dapla_metadata._shared import config
+from dapla_metadata._shared.enums import DaplaRegion
+from dapla_metadata._shared.enums import DaplaService
 
 logger = logging.getLogger(__name__)
 
@@ -86,3 +86,30 @@ def get_user_info_for_current_platform() -> UserInfo:
             "Was not possible to retrieve user information! Some fields may not be set.",
         )
         return UnknownUserInfo()
+
+
+def get_owner() -> str:
+    """Returns the owner read from the GROUP_CONTEXT environment variable."""
+    if group := config.get_dapla_group_context():
+        return parse_team_name(group)
+    msg = "DAPLA_GROUP_CONTEXT environment variable not found"
+    raise OSError(msg)
+
+
+def parse_team_name(group: str) -> str:
+    """Parses the group to get the current team.
+
+    >>> parse_team_name(dapla-metadata-developers)
+    (dapla-metadata)
+
+    >>> parse_team_name(dapla-metadata-data-admins)
+    (dapla-metadata)
+
+    >>> parse_team_name(dapla-metadata)
+    (dapla)
+
+    >>> parse_team_name(dapla-metadata-not-real-name)
+    (dapla-metadata-not-real)
+    """
+    parts = group.split("-")
+    return "-".join(parts[:-2] if group.endswith("data-admins") else parts[:-1])

{dapla_toolbelt_metadata-0.2.6 → dapla_toolbelt_metadata-0.4.0}/src/dapla_metadata/datasets/core.py

@@ -13,8 +13,8 @@ from typing import TYPE_CHECKING
 from datadoc_model import model
 from datadoc_model.model import DataSetStatus
 
-from dapla_metadata.datasets import config
-from dapla_metadata.datasets import user_info
+from dapla_metadata._shared import config
+from dapla_metadata._shared import user_info
 from dapla_metadata.datasets.dapla_dataset_path_info import DaplaDatasetPathInfo
 from dapla_metadata.datasets.dataset_parser import DatasetParser
 from dapla_metadata.datasets.model_backwards_compatibility import (
@@ -42,6 +42,7 @@ from dapla_metadata.datasets.utility.utils import (
     num_obligatory_variables_fields_completed,
 )
 from dapla_metadata.datasets.utility.utils import override_dataset_fields
+from dapla_metadata.datasets.utility.utils import set_dataset_owner
 from dapla_metadata.datasets.utility.utils import set_default_values_dataset
 from dapla_metadata.datasets.utility.utils import set_default_values_variables
 
@@ -189,6 +190,7 @@ class Datadoc:
             self._set_metadata(existing_metadata or extracted_metadata)
         set_default_values_variables(self.variables)
         set_default_values_dataset(self.dataset)
+        set_dataset_owner(self.dataset)
         self._create_variables_lookup()
 
     def _get_existing_file_path(

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/datasets/utility/enums.py

@@ -0,0 +1,16 @@
+"""Enumerations used in Datadoc."""
+
+from __future__ import annotations
+
+from enum import Enum
+
+
+class SupportedLanguages(str, Enum):
+    """The list of languages metadata may be recorded in.
+
+    Reference: https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
+    """
+
+    NORSK_BOKMÅL = "nb"
+    NORSK_NYNORSK = "nn"
+    ENGLISH = "en"

{dapla_toolbelt_metadata-0.2.6 → dapla_toolbelt_metadata-0.4.0}/src/dapla_metadata/datasets/utility/utils.py

@@ -14,6 +14,7 @@ from datadoc_model.model import Assessment
 from datadoc_model.model import DataSetState
 from datadoc_model.model import VariableRole
 
+from dapla_metadata._shared import user_info
 from dapla_metadata.datasets.utility.constants import (
     DATASET_FIELDS_FROM_EXISTING_METADATA,
 )
@@ -139,6 +140,18 @@ def set_default_values_dataset(dataset: model.Dataset) -> None:
         dataset.contains_personal_data = False
 
 
+def set_dataset_owner(dataset: model.Dataset) -> None:
+    """Sets the owner of the dataset from the DAPLA_GROUP_CONTEXT enviornment variable.
+
+    Args:
+        dataset: The dataset object to set default values on.
+    """
+    try:
+        dataset.owner = user_info.get_owner()
+    except OSError:
+        logger.exception("Failed to find environment variable DAPLA_GROUP_CONTEXT")
+
+
 def set_variables_inherit_from_dataset(
     dataset: model.Dataset,
     variables: list,

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/variable_definitions/__init__.py

@@ -0,0 +1,7 @@
+"""Client for working with Variable Definitions at Statistics Norway."""
+
+from .generated.vardef_client import models
+from .generated.vardef_client.exceptions import *  # noqa: F403
+from .vardef import Vardef
+from .variable_definition import CompletePatchOutput
+from .variable_definition import VariableDefinition

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/variable_definitions/_client.py

@@ -0,0 +1,30 @@
+from dapla_metadata.variable_definitions.config import get_vardef_client_configuration
+from dapla_metadata.variable_definitions.generated.vardef_client.api_client import (
+    ApiClient,
+)
+from dapla_metadata.variable_definitions.generated.vardef_client.configuration import (
+    Configuration,
+)
+
+
+class VardefClient:
+    _client: ApiClient | None = None
+    _config: Configuration | None = None
+
+    @classmethod
+    def get_config(cls) -> Configuration | None:
+        """Get the client configuration object."""
+        return cls._config
+
+    @classmethod
+    def set_config(cls, config: Configuration) -> None:
+        """Set the client configuration object."""
+        cls._config = config
+
+    @classmethod
+    def get_client(cls) -> ApiClient:
+        if not cls._client:
+            if not cls._config:
+                cls._config = get_vardef_client_configuration()
+            cls._client = ApiClient(cls._config)
+        return cls._client

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/variable_definitions/config.py

@@ -0,0 +1,49 @@
+from dapla_metadata._shared.config import get_config_item
+from dapla_metadata._shared.config import get_dapla_environment
+from dapla_metadata._shared.config import get_dapla_group_context
+from dapla_metadata._shared.config import get_oidc_token
+from dapla_metadata._shared.enums import DaplaEnvironment
+from dapla_metadata.variable_definitions.generated.vardef_client.configuration import (
+    Configuration,
+)
+
+VARDEF_HOST_TEST = "https://metadata.intern.test.ssb.no"
+
+
+def get_active_group() -> str:
+    """Get the group the user currently represents.
+
+    Returns:
+        A string with the active group.
+    """
+    return str(get_dapla_group_context(raising=True))
+
+
+def get_vardef_host() -> str:
+    """Get the host for the server to make requests to.
+
+    Based on standard Dapla platform environment variables, or variables specific
+    to Vardef if these are not available.
+
+    Raises:
+        NotImplementedError: When Vardef is not available for the current environment.
+    """
+    match get_dapla_environment():
+        case DaplaEnvironment.PROD:
+            msg = "Vardef is not available in prod."
+            raise NotImplementedError(msg)
+        case DaplaEnvironment.TEST:
+            return VARDEF_HOST_TEST
+        case DaplaEnvironment.DEV:
+            msg = "Vardef is not available in dev."
+            raise NotImplementedError(msg)
+        case _:
+            return get_config_item("VARDEF_HOST") or "http://localhost:8080"
+
+
+def get_vardef_client_configuration() -> Configuration:
+    """Build a config to be supplied to the `ApiClient`."""
+    return Configuration(
+        host=get_vardef_host(),
+        access_token=get_oidc_token(raising=True),
+    )

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/variable_definitions/exceptions.py

@@ -0,0 +1,66 @@
+"""Vardef client exceptions."""
+
+import json
+from functools import wraps
+
+from dapla_metadata.variable_definitions.generated.vardef_client.exceptions import (
+    OpenApiException,
+)
+
+
+class VardefClientException(OpenApiException):
+    """Custom exception to represent errors encountered in the Vardef client.
+
+    This exception extracts and formats error details from a JSON response body
+    provided by the Vardef API, enabling more descriptive error messages.
+    If the response body cannot be parsed as JSON or lacks expected keys,
+    default values are used to provide meaningful feedback.
+    """
+
+    def __init__(self, response_body: str) -> None:
+        """Initialize the exception with a JSON-formatted response body.
+
+        Args:
+            response_body (str): The JSON string containing error details
+                                from the Vardef API response.
+
+        Attributes:
+            status (str): The status code from the response, or "Unknown status"
+                        if not available or the response is invalid.
+            detail (str || list): A detailed error message from the response, or
+                        "No detail provided" if not provided. If "Constraint violation"
+                        detail contains 'field' and 'message'.
+            response_body (str): The raw response body string, stored for
+                                debugging purposes.
+        """
+        try:
+            data = json.loads(response_body)
+            self.status = data.get("status", "Unknown status")
+            if data.get("title") == "Constraint Violation":
+                violations = data.get("violations", [])
+                self.detail = "".join(
+                    f"\n{violation.get('field', 'Unknown field')}: {violation.get('message', 'No message provided')}"
+                    for violation in violations
+                )
+
+            else:
+                self.detail = data.get("detail", "No detail provided")
+            self.response_body = response_body
+        except (json.JSONDecodeError, TypeError):
+            self.status = "Unknown"
+            self.detail = "Could not decode error response from API"
+            data = None
+        super().__init__(f"Status {self.status}: {self.detail}")
+
+
+def vardef_exception_handler(method):  # noqa: ANN201, ANN001
+    """Decorator for handling exceptions in Vardef."""
+
+    @wraps(method)
+    def _impl(self, *method_args, **method_kwargs):  # noqa: ANN001, ANN002, ANN003
+        try:
+            return method(self, *method_args, **method_kwargs)
+        except OpenApiException as e:
+            raise VardefClientException(e.body) from e
+
+    return _impl

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/variable_definitions/generated/.openapi-generator/FILES

@@ -0,0 +1,22 @@
+vardef_client/api/__init__.py
+vardef_client/api/data_migration_api.py
+vardef_client/api/draft_variable_definitions_api.py
+vardef_client/api/patches_api.py
+vardef_client/api/public_api.py
+vardef_client/api/validity_periods_api.py
+vardef_client/api/variable_definitions_api.py
+vardef_client/models/__init__.py
+vardef_client/models/complete_response.py
+vardef_client/models/contact.py
+vardef_client/models/draft.py
+vardef_client/models/klass_reference.py
+vardef_client/models/language_string_type.py
+vardef_client/models/owner.py
+vardef_client/models/patch.py
+vardef_client/models/problem.py
+vardef_client/models/rendered_contact.py
+vardef_client/models/rendered_variable_definition.py
+vardef_client/models/supported_languages.py
+vardef_client/models/update_draft.py
+vardef_client/models/validity_period.py
+vardef_client/models/variable_status.py

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/variable_definitions/generated/.openapi-generator/VERSION

@@ -0,0 +1 @@
+7.9.0

dapla_toolbelt_metadata-0.4.0/src/dapla_metadata/variable_definitions/generated/.openapi-generator-ignore

@@ -0,0 +1,6 @@
+# Ignore everything to make sure we don't get extra cruft
+**
+# Unignore the directories with domain code
+!vardef_client/api/
+!vardef_client/docs/
+!vardef_client/models/