dapla-toolbelt-metadata 0.2.1__py3-none-any.whl → 0.9.11__py3-none-any.whl

dapla_metadata/variable_definitions/exceptions.py

@@ -0,0 +1,255 @@
+"""Vardef client exceptions."""
+
+import json
+from functools import wraps
+from http import HTTPStatus
+from types import MappingProxyType
+
+import urllib3
+from pytz import UnknownTimeZoneError
+from ruamel.yaml.error import YAMLError
+
+from dapla_metadata.variable_definitions._generated.vardef_client.exceptions import (
+    OpenApiException,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.exceptions import (
+    UnauthorizedException,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.variable_status import (
+    VariableStatus,
+)
+from dapla_metadata.variable_definitions._utils._client import VardefClient
+from dapla_metadata.variable_definitions._utils.constants import (
+    PUBLISHING_BLOCKED_ERROR_MESSAGE,
+)
+from dapla_metadata.variable_definitions._utils.constants import VARDEF_PROD_URL
+
+# Use MappingProxyType so the dict is immutable
+STATUS_EXPLANATIONS: MappingProxyType[HTTPStatus | None, str] = MappingProxyType(
+    {
+        HTTPStatus.BAD_REQUEST: "There was a problem with the supplied data. Please review the data you supplied based on the details below.",
+        HTTPStatus.UNAUTHORIZED: "There is a problem with your token, it may have expired. Try logging out, logging in and trying again.",
+        HTTPStatus.FORBIDDEN: "Forbidden. You don't have access to this, possibly because your team is not an owner of this variable definition.",
+        HTTPStatus.NOT_FOUND: "The variable definition was not found, check that the `short_name` or `id` is correct and that the variable exists.",
+        HTTPStatus.METHOD_NOT_ALLOWED: "That won't work here. The status of your variable definition likely doesn't allow for this to happen.",
+        HTTPStatus.CONFLICT: "There was a conflict with existing data. Please change your data and try again.",
+        None: "",
+    },
+)
+
+
+class VardefClientError(Exception):
+    """Custom exception to represent errors encountered in the Vardef client.
+
+    This exception extracts and formats error details from a JSON response body
+    provided by the Vardef API, enabling more descriptive error messages.
+    If the response body cannot be parsed as JSON or lacks expected keys,
+    default values are used to provide meaningful feedback.
+    """
+
+    def __init__(self, response_body: str) -> None:
+        """Initialize the exception with a JSON-formatted response body.
+
+        Args:
+            response_body (str): The JSON string containing error details
+                                from the Vardef API response.
+
+        Attributes:
+            status (str): The status code from the response, or "Unknown status"
+                        if not available or the response is invalid.
+            detail (str || list): A detailed error message from the response, or
+                        "No detail provided" if not provided. If "Constraint violation"
+                        detail contains 'field' and 'message'.
+            response_body (str): The raw response body string, stored for
+                                debugging purposes.
+        """
+        self.status: int | None = None
+        self.detail: str = ""
+        try:
+            data = json.loads(response_body)
+            self.status = data.get("status")
+            if data.get("title") == "Constraint Violation":
+                violations = data.get("violations", [])
+                self.detail = "\n" + "\n".join(
+                    f"{violation.get('field', 'Unknown field')}: {violation.get('message', 'No message provided')}"
+                    for violation in violations
+                )
+            else:
+                self.detail = data.get("detail")
+
+            if self.detail:
+                self.detail = f"\nDetail: {self.detail}"
+            self.response_body = response_body
+        except (json.JSONDecodeError, TypeError):
+            self.detail = "Could not decode error response from API"
+
+        super().__init__(
+            f"{self._get_status_explanation(self.status)}{self.detail if self.detail else ''}",
+        )
+
+    @staticmethod
+    def _get_status_explanation(status: int | None) -> str:
+        return (
+            ""
+            if not status
+            else (STATUS_EXPLANATIONS.get(HTTPStatus(status)) or f"Status {status}:")
+        )
+
+
+def vardef_exception_handler(method):  # noqa: ANN201, ANN001
+    """Decorator for handling exceptions in Vardef."""
+
+    @wraps(method)
+    def _impl(self, *method_args, **method_kwargs):  # noqa: ANN001, ANN002, ANN003
+        try:
+            return method(self, *method_args, **method_kwargs)
+        except urllib3.exceptions.HTTPError as e:
+            # Catch all urllib3 exceptions by catching the base class.
+            # These exceptions typically arise from lower level network problems.
+            raise VardefClientError(
+                json.dumps(
+                    {
+                        "status": None,
+                        "title": "Network problems",
+                        "detail": f"""There was a network problem when sending the request to the server. Try again shortly.
+Original exception:
+{getattr(e, "message", repr(e))}""",
+                    },
+                ),
+            ) from e
+        except UnauthorizedException as e:
+            raise VardefClientError(
+                json.dumps(
+                    {
+                        "status": e.status,
+                        "title": "Unauthorized",
+                        "detail": "Unauthorized",
+                    },
+                ),
+            ) from e
+        except OpenApiException as e:
+            raise VardefClientError(e.body) from e
+
+    return _impl
+
+
+class VariableNotFoundError(Exception):
+    """Custom exception for when a variable is not found.
+
+    Attributes:
+        message (str): Message describing the error.
+    """
+
+    def __init__(self, message: str) -> None:
+        """Initialize the VariableNotFoundError.
+
+        Args:
+            message (str): Description of the error.
+        """
+        super().__init__(message)
+        self.message = message
+
+    def __str__(self) -> str:
+        """Return the string representation of the exception."""
+        return f" {self.message}"
+
+
+class VardefFileError(Exception):
+    """Custom exception for catching errors related to variable definition file handling.
+
+    Attributes:
+        message (str): Message describing the error.
+    """
+
+    def __init__(self, message: str, *args) -> None:  # noqa: ANN002
+        """Accepting the message and any additional arguments."""
+        super().__init__(message, *args)
+        self.message = message
+        self.args = args
+
+    def __str__(self) -> str:
+        """Returning a custom string representation of the exception."""
+        return f"VardefFileError: {self.message}"
+
+
+def vardef_file_error_handler(method):  # noqa: ANN201, ANN001
+    """Decorator for handling exceptions when generating yaml files for variable definitions."""
+
+    @wraps(method)
+    def _impl(*method_args, **method_kwargs):  # noqa: ANN002, ANN003
+        try:
+            return method(
+                *method_args,
+                **method_kwargs,
+            )
+        except FileExistsError as e:
+            msg = f"File already exists and can not be saved: {method_kwargs.get('file_path', 'unknown file path')}"
+            raise VardefFileError(msg) from e
+        except PermissionError as e:
+            msg = f"Permission denied for file path when accessing the file. Original error: {e!s}"
+            raise VardefFileError(msg) from e
+        except UnknownTimeZoneError as e:
+            msg = f"Timezone is unknown: {method_kwargs.get('time_zone', 'unknown')}"
+            raise VardefFileError(msg) from e
+        except YAMLError as e:
+            msg = f"Invalid yaml. Please fix the formatting in your yaml file.\nOriginal error:\n{e!s}"
+            raise VardefFileError(msg) from e
+        except EOFError as e:
+            msg = "Unexpected end of file"
+            raise VardefFileError(msg) from e
+        except NotADirectoryError as e:
+            msg = f"Path is not a directory: {method_kwargs.get('file_path', 'unknown file path')}. Original error: {e!s}"
+            raise VardefFileError(msg) from e
+
+    return _impl
+
+
+class PublishingBlockedError(RuntimeError):
+    """Exception raised when publishing variable definitions is blocked in the production environment."""
+
+    default_message = PUBLISHING_BLOCKED_ERROR_MESSAGE
+
+    def __str__(self) -> str:
+        """Return the default publishing blocked error message."""
+        return self.default_message
+
+
+def publishing_blocked_error_handler(method):  # noqa: ANN201, ANN001
+    """Decorator that blocks publishing variable definitions in production.
+
+    - If the environment is production:
+        - If `variable_status` is present in the arguments and set to `PUBLISHED_INTERNAL` or `PUBLISHED_EXTERNAL`,
+        publishing is blocked by raising a `PublishingBlockedError`.
+        - If `variable_status` is set to `DRAFT`, the method is allowed to proceed.
+        - If no arguments are provided, all publishing attempts are blocked.
+    """
+
+    @wraps(method)
+    def _impl(*method_args, **method_kwargs):  # noqa: ANN002, ANN003
+        try:
+            client = VardefClient()
+            host = client.get_config().host
+        except VardefClientError as e:
+            msg = f"Failed to get VardefClient config: {e}"
+            raise RuntimeError(msg) from e
+
+        if host == VARDEF_PROD_URL:
+            if len(method_args) > 1:
+                status = method_args[1].variable_status
+                if status == VariableStatus.DRAFT:
+                    return method(
+                        *method_args,
+                        **method_kwargs,
+                    )
+                if status in (
+                    VariableStatus.PUBLISHED_INTERNAL,
+                    VariableStatus.PUBLISHED_EXTERNAL,
+                ):
+                    raise PublishingBlockedError
+            raise PublishingBlockedError
+        return method(
+            *method_args,
+            **method_kwargs,
+        )
+
+    return _impl

dapla_metadata/variable_definitions/vardef.py

@@ -0,0 +1,372 @@
+import logging
+from datetime import date
+from os import PathLike
+from pathlib import Path
+
+from dapla_metadata.variable_definitions._generated.vardef_client.api.data_migration_api import (
+    DataMigrationApi,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.api.draft_variable_definitions_api import (
+    DraftVariableDefinitionsApi,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.api.variable_definitions_api import (
+    VariableDefinitionsApi,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
+    CompleteResponse,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.draft import (
+    Draft,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.vardok_id_response import (
+    VardokIdResponse,
+)
+from dapla_metadata.variable_definitions._utils import config
+from dapla_metadata.variable_definitions._utils._client import VardefClient
+from dapla_metadata.variable_definitions._utils.template_files import (
+    _find_latest_template_file,
+)
+from dapla_metadata.variable_definitions._utils.template_files import (
+    create_template_yaml,
+)
+from dapla_metadata.variable_definitions._utils.variable_definition_files import (
+    _read_file_to_model,
+)
+from dapla_metadata.variable_definitions.exceptions import VariableNotFoundError
+from dapla_metadata.variable_definitions.exceptions import vardef_exception_handler
+from dapla_metadata.variable_definitions.exceptions import vardef_file_error_handler
+from dapla_metadata.variable_definitions.vardok_id import VardokId
+from dapla_metadata.variable_definitions.vardok_vardef_id_pair import VardokVardefIdPair
+from dapla_metadata.variable_definitions.variable_definition import VariableDefinition
+
+logger = logging.getLogger(__name__)
+
+
+class Vardef:
+    """Create, maintain and read Variable Definitions.
+
+    ====================
+    Variable Definitions
+    ====================
+
+    Variable Definitions are centralized definitions of concrete variables which are typically present in multiple datasets. Variable Definitions
+    support standardization of data and metadata and facilitate sharing and joining of data by clarifying when variables have an identical
+    definition.
+
+    The methods in this class allow for creation, maintenance and access of Variable Definitions.
+
+    Creation and maintenance of variables may only be performed by Statistics Norway employees representing a specific Dapla team, who are defined
+    as the owners of a given Variable Definition. The group a user represents is chosen when starting a service in Dapla Lab. This class will
+    seamlessly detect this and send it to the API. All maintenance is to be performed by the owners, with no intervention from administrators.
+
+    ======
+    Status
+    ======
+    All Variable Definitions have an associated status. The possible values for status are `DRAFT`, `PUBLISHED_INTERNAL` and `PUBLISHED_EXTERNAL`.
+
+    -----
+    Draft
+    -----
+    When a Variable Definition is created it is assigned the status `DRAFT`. Under this status the Variable Definition is:
+    * Only visible to Statistics Norway employees.
+    * Mutable (it may be changed directly without need for versioning).
+    * Not suitable to refer to from other systems.
+    This status may be changed to `PUBLISHED_INTERNAL` or `PUBLISHED_EXTERNAL` with a direct update.
+
+    ------------------
+    Published Internal
+    ------------------
+    Under this status the Variable Definition is:
+    * Only visible to Statistics Norway employees.
+    * Immutable (all changes are versioned).
+    * Suitable to refer to in internal systems for statistics production.
+    * Not suitable to refer to for external use (for example in Statistikkbanken).
+
+    This status may be changed to `PUBLISHED_EXTERNAL` by creating a Patch version.
+
+    ------------------
+    Published External
+    ------------------
+    Under this status the Variable Definition is:
+    * Visible to the general public.
+    * Immutable (all changes are versioned).
+    * Suitable to refer to from any system.
+
+    This status may not be changed as it would break immutability. If a Variable Definition is no longer relevant then its period of validity
+    should be ended by specifying a `valid_until` date in a Patch version.
+
+    ============
+    Immutability
+    ============
+    Variable Definitions are immutable. This means that any changes must be performed in a strict versioning system. Consumers can avoid
+    being exposed to breaking changes by specifying a `date_of_validity` when they request a Variable Definition.
+    """
+
+    @classmethod
+    @vardef_exception_handler
+    def create_draft(cls, draft: Draft) -> VariableDefinition:
+        """Create a Draft Variable Definition."""
+        new_variable = VariableDefinition.from_model(
+            DraftVariableDefinitionsApi(
+                VardefClient.get_client(),
+            ).create_variable_definition(
+                active_group=config.get_active_group(),
+                draft=draft,
+            ),
+        )
+
+        logger.info(
+            "✅ Successfully created variable definition '%s' with ID '%s'",
+            new_variable.short_name,
+            new_variable.id,
+        )
+        return new_variable
+
+    @classmethod
+    @vardef_file_error_handler
+    def create_draft_from_file(
+        cls,
+        file_path: PathLike[str] | None = None,
+    ) -> VariableDefinition:
+        """Create a Draft Variable Definition from a stored yaml file.
+
+        By default the latest template file in the default directory is chosen, this may be overridden by providing a value for the optional `file_path` parameter.
+
+        Args:
+            file_path (PathLike[str], optional): Supply a file path to override the automatic one. Defaults to None.
+
+        Raises:
+            FileNotFoundError: When a file can't be found.
+
+        Returns:
+            VariableDefinition: The created draft variable definition.
+        """
+        return cls.create_draft(
+            _read_file_to_model(
+                file_path or _find_latest_template_file(),
+                Draft,
+            ),
+        )
+
+    @classmethod
+    @vardef_exception_handler
+    def migrate_from_vardok(cls, vardok_id: str) -> VariableDefinition:
+        """Migrate a Variable Definition from Vardok to Vardef.
+
+        - Each Vardok Variable Definition may only be migrated once.
+        - The Dapla team of the person who performs the migration will be set as the owner.
+        - All metadata should be checked for correctness before publication.
+
+        Args:
+            vardok_id (str): The ID of a Variable Definition in Vardok.
+
+        Returns:
+            VariableDefinition: The migrated Variable Definition in Vardef.
+        """
+        migrated_variable = VariableDefinition.from_model(
+            DataMigrationApi(
+                VardefClient.get_client(),
+            ).create_variable_definition_from_var_dok(
+                active_group=config.get_active_group(),
+                vardok_id=vardok_id,
+            ),
+        )
+
+        logger.info(
+            "✅ Successfully migrated variable definition '%s' with ID '%s'",
+            migrated_variable.short_name,
+            migrated_variable.id,
+        )
+        return migrated_variable
+
+    @classmethod
+    @vardef_exception_handler
+    def list_vardok_vardef_mapping(cls) -> list[VardokVardefIdPair]:
+        """List the mapping between vardok and vardef.
+
+        Returns:
+            List[VardokVardefIdPair]: The list with mappings between Vardok and Vardef
+        """
+        return [
+            VardokVardefIdPair.from_model(definition)
+            for definition in DataMigrationApi(
+                VardefClient.get_client(),
+            ).get_vardok_vardef_mapping()
+        ]
+
+    @classmethod
+    @vardef_exception_handler
+    def get_variable_definition_by_vardok_id(
+        cls,
+        vardok_id: str,
+    ) -> VariableDefinition:
+        """Get a Variable Definition by its Vardok ID.
+
+        Args:
+            vardok_id (str): The Vardok ID of the desired Variable Definition
+
+        Returns:
+            VariableDefinition: The Variable Definition.
+
+        Raises:
+            TypeError: If the incorrect type is returned.
+        """
+        raw_response = DataMigrationApi(
+            VardefClient.get_client()
+        ).get_vardok_vardef_mapping_by_id(
+            vardok_id,
+        )
+
+        if isinstance(raw_response.actual_instance, CompleteResponse):
+            return VariableDefinition.from_model(raw_response.actual_instance)
+        msg = "Unexpected response type"
+        raise TypeError(msg)
+
+    @classmethod
+    @vardef_exception_handler
+    def get_vardok_id_by_short_name(
+        cls,
+        short_name: str,
+    ) -> VardokId:
+        """Retrieve a Vardok id by short name.
+
+        Args:
+            short_name (str): The short name of the desired Variable Definition
+
+        Raises:
+            TypeError: If the incorrect type is returned.
+        """
+        variable_definition = cls.get_variable_definition_by_shortname(short_name)
+
+        raw_response = DataMigrationApi(
+            VardefClient.get_client()
+        ).get_vardok_vardef_mapping_by_id(
+            variable_definition.id,
+        )
+
+        if isinstance(raw_response.actual_instance, VardokIdResponse):
+            return VardokId.from_model(raw_response.actual_instance)
+        msg = "Unexpected response type"
+        raise TypeError(msg)
+
+    @classmethod
+    @vardef_exception_handler
+    def list_variable_definitions(
+        cls,
+        date_of_validity: date | None = None,
+    ) -> list[VariableDefinition]:
+        """List variable definitions.
+
+        ---------
+        Filtering
+        ---------
+        If no filter arguments are provided then all Variable Definitions are returned. See the documentation for the
+        individual arguments to understand their effect. Filter arguments are combined with AND logic.
+
+        Args:
+            date_of_validity (date | None, optional): List only variable definitions which are valid on this date. Defaults to None.
+
+        Returns:
+            list[VariableDefinition]: The list of Variable Definitions.
+        """
+        return [
+            VariableDefinition.from_model(definition)
+            for definition in VariableDefinitionsApi(
+                VardefClient.get_client(),
+            ).list_variable_definitions(
+                date_of_validity=date_of_validity,
+            )
+        ]
+
+    @classmethod
+    @vardef_exception_handler
+    def get_variable_definition_by_id(
+        cls,
+        variable_definition_id: str,
+        date_of_validity: date | None = None,
+    ) -> VariableDefinition:
+        """Get a Variable Definition by ID.
+
+        Args:
+            variable_definition_id (str): The ID of the desired Variable Definition
+            date_of_validity (date | None, optional): List only variable definitions which are valid on this date. Defaults to None.
+
+        Returns:
+            VariableDefinition: The Variable Definition.
+
+        Raises:
+            NotFoundException when the given ID is not found
+        """
+        return VariableDefinition.from_model(
+            VariableDefinitionsApi(
+                VardefClient.get_client(),
+            ).get_variable_definition_by_id(
+                variable_definition_id=variable_definition_id,
+                date_of_validity=date_of_validity,
+            ),
+        )
+
+    @classmethod
+    @vardef_exception_handler
+    def get_variable_definition_by_shortname(
+        cls,
+        short_name: str,
+        date_of_validity: date | None = None,
+    ) -> VariableDefinition:
+        """Retrieve a Variable Definition by short name.
+
+        Args:
+            short_name (str): The short name of the Variable Definition.
+            date_of_validity (date | None, optional): Filter by validity date. Defaults to None.
+
+        Returns:
+            VariableDefinition: The retrieved Variable Definition.
+
+        Raises:
+            VariableNotFoundError: If no matching Variable Definition is found.
+            ValueError: If multiple variables with the same shortname is found.
+        """
+        client = VardefClient.get_client()
+        api = VariableDefinitionsApi(client)
+
+        variable_definitions = api.list_variable_definitions(
+            short_name=short_name,
+            date_of_validity=date_of_validity,
+        )
+
+        if not variable_definitions:
+            msg = f"Variable with short name {short_name} not found"
+            raise VariableNotFoundError(msg)
+        if len(variable_definitions) > 1:
+            msg = f"Lookup by short name {short_name} found multiple variables which should not be possible."
+            raise VariableNotFoundError(msg)
+
+        return VariableDefinition.from_model(variable_definitions[0])
+
+    @classmethod
+    @vardef_file_error_handler
+    def write_template_to_file(cls, custom_file_path: str | None = None) -> Path:
+        """Write template with default values to a yaml file."""
+        file_path = create_template_yaml(
+            custom_directory=Path(custom_file_path) if custom_file_path else None,
+        )
+        logger.info(
+            f"✅ Created editable variable definition template file at {file_path}",  # noqa: G004
+        )
+        return file_path
+
+    @classmethod
+    @vardef_exception_handler
+    def does_short_name_exist(
+        cls,
+        short_name: str,
+    ) -> bool:
+        """Return True if the short name exists in Vardef, otherwise False."""
+        variable_definitions = Vardef.list_variable_definitions()
+        for variable in variable_definitions:
+            if short_name.strip() == variable.short_name:
+                logger.info(
+                    f"Found duplicate short name {short_name}",  # noqa: G004
+                )
+                return True
+        return False

dapla_metadata/variable_definitions/vardok_id.py

@@ -0,0 +1,48 @@
+import logging
+from pathlib import Path
+
+from pydantic import ConfigDict
+from pydantic import PrivateAttr
+
+from dapla_metadata.variable_definitions._generated.vardef_client.models.vardok_id_response import (
+    VardokIdResponse,
+)
+from dapla_metadata.variable_definitions._utils.variable_definition_files import (
+    _convert_to_yaml_output,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class VardokId(VardokIdResponse):
+    """A Vardok id.
+
+    - Provides access to the Vardok id filed.
+    - Provides methods allowing maintenance for nicer output of the Vardok id.
+
+    Args:
+        VardokIdResponse: The Pydantic model superclass, representing a Vardok id response.
+    """
+
+    _file_path: Path | None = PrivateAttr(None)
+
+    model_config = ConfigDict(use_enum_values=True, str_strip_whitespace=True)
+
+    @staticmethod
+    def from_model(
+        model: VardokIdResponse,
+    ) -> "VardokId":
+        """Create a VariableDefinition instance from a CompleteResponse."""
+        self = VardokId.from_dict(model.model_dump())
+        if not self:
+            msg = f"Could not construct a VardokId instance from {model}"
+            raise ValueError(msg)
+        return self
+
+    def __str__(self) -> str:
+        """Format as indented YAML."""
+        return _convert_to_yaml_output(self)
+
+    def __repr__(self) -> str:
+        """Format as indented YAML."""
+        return _convert_to_yaml_output(self)