dapla-toolbelt-metadata 0.4.2__py3-none-any.whl → 0.5.0__py3-none-any.whl

dapla_metadata/variable_definitions/_utils/files.py

@@ -0,0 +1,273 @@
+"""Lower level file utilities."""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+from typing import cast
+
+import pytz
+from pydantic.config import JsonDict
+from ruamel.yaml import YAML
+from ruamel.yaml import CommentedMap
+
+from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
+    CompleteResponse,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.variable_status import (
+    VariableStatus,
+)
+from dapla_metadata.variable_definitions._utils import config
+from dapla_metadata.variable_definitions._utils.constants import (
+    MACHINE_GENERATED_FIELDS,
+)
+from dapla_metadata.variable_definitions._utils.constants import NORWEGIAN_DESCRIPTIONS
+from dapla_metadata.variable_definitions._utils.constants import OPTIONAL_FIELD
+from dapla_metadata.variable_definitions._utils.constants import OWNER_FIELD_NAME
+from dapla_metadata.variable_definitions._utils.constants import REQUIRED_FIELD
+from dapla_metadata.variable_definitions._utils.constants import (
+    TEMPLATE_SECTION_HEADER_MACHINE_GENERATED,
+)
+from dapla_metadata.variable_definitions._utils.constants import (
+    TEMPLATE_SECTION_HEADER_OWNER,
+)
+from dapla_metadata.variable_definitions._utils.constants import (
+    TEMPLATE_SECTION_HEADER_STATUS,
+)
+from dapla_metadata.variable_definitions._utils.constants import (
+    VARIABLE_DEFINITIONS_DIR,
+)
+from dapla_metadata.variable_definitions._utils.constants import (
+    VARIABLE_STATUS_FIELD_NAME,
+)
+from dapla_metadata.variable_definitions._utils.descriptions import (
+    apply_norwegian_descriptions_to_model,
+)
+from dapla_metadata.variable_definitions.exceptions import VardefFileError
+
+if TYPE_CHECKING:
+    from pydantic import JsonValue
+
+logger = logging.getLogger(__name__)
+
+
+def _create_file_name(
+    base_name: str,
+    time_object: str,
+    short_name: str | None = None,
+    variable_definition_id: str | None = None,
+) -> str:
+    """Return file name with dynamic timestamp, and shortname and id if available."""
+    return (
+        "_".join(
+            filter(
+                None,
+                [
+                    base_name,
+                    short_name,
+                    variable_definition_id,
+                    time_object,
+                ],
+            ),
+        )
+        + ".yaml"
+    )
+
+
+def _get_current_time() -> str:
+    """Return a string format date now for filename."""
+    timezone = pytz.timezone("Europe/Oslo")
+    current_datetime = datetime.now(timezone).strftime("%Y-%m-%dT%H-%M-%S")
+    return str(current_datetime)
+
+
+def _get_workspace_dir() -> Path:
+    """Determine the workspace directory."""
+    workspace_dir = config.get_workspace_dir()
+
+    if workspace_dir is None:
+        msg = "WORKSPACE_DIR is not set. Check your configuration or provide a custom directory."
+        raise VardefFileError(msg)
+    workspace_dir_path: Path
+    if workspace_dir is not None:
+        workspace_dir_path = Path(workspace_dir)
+        workspace_dir_path.resolve()
+
+        if not workspace_dir_path.exists():
+            msg = f"Directory '{workspace_dir_path}' does not exist."
+            raise FileNotFoundError(msg)
+
+        if not workspace_dir_path.is_dir():
+            msg = f"'{workspace_dir_path}' is not a directory."
+            raise NotADirectoryError(msg)
+        logger.debug("'WORKSPACE_DIR' value: %s", workspace_dir)
+    return workspace_dir_path
+
+
+def _get_variable_definitions_dir():
+    """Get or create the variable definitions directory inside the workspace."""
+    workspace_dir = _get_workspace_dir()
+    folder_path = workspace_dir / VARIABLE_DEFINITIONS_DIR
+    folder_path.mkdir(parents=True, exist_ok=True)
+    return folder_path
+
+
+def _populate_commented_map(
+    field_name: str,
+    value: str,
+    commented_map: CommentedMap,
+    model_instance: CompleteResponse,
+) -> None:
+    """Add data to a CommentedMap."""
+    commented_map[field_name] = value
+    field = model_instance.model_fields[field_name]
+    description: JsonValue = cast(
+        JsonDict,
+        field.json_schema_extra,
+    )[NORWEGIAN_DESCRIPTIONS]
+    if description is not None:
+        new_description = (
+            (REQUIRED_FIELD if field.is_required() else OPTIONAL_FIELD)
+            + "\n"
+            + str(description)
+        )
+        commented_map.yaml_set_comment_before_after_key(
+            field_name,
+            before=new_description,
+        )
+
+
+def _validate_and_create_directory(custom_directory: Path) -> Path:
+    """Ensure that the given path is a valid directory, creating it if necessary.
+
+    Args:
+        custom_directory (Path): The target directory path.
+
+    Returns:
+        Path: The resolved absolute path of the directory.
+
+    Raises:
+        ValueError: If the provided path has a file suffix, indicating a file name instead of a directory.
+        NotADirectoryError: If the path exists but is not a directory.
+        PermissionError: If there are insufficient permissions to create the directory.
+        OSError: If an OS-related error occurs while creating the directory.
+    """
+    custom_directory = Path(custom_directory).resolve()
+
+    if custom_directory.suffix:
+        msg = f"Expected a directory but got a file name: %{custom_directory.name}"
+        raise ValueError(msg)
+
+    if custom_directory.exists() and not custom_directory.is_dir():
+        msg = f"Path exists but is not a directory: {custom_directory}"
+        raise NotADirectoryError(msg)
+
+    try:
+        custom_directory.mkdir(parents=True, exist_ok=True)
+    except PermissionError as e:
+        msg = f"Insufficient permissions to create directory: {custom_directory}"
+        raise PermissionError(msg) from e
+    except OSError as e:
+        msg = f"Failed to create directory {custom_directory}: {e!s}"
+        raise OSError(msg) from e
+
+    return custom_directory
+
+
+def _configure_yaml() -> YAML:
+    yaml = YAML()  # Use ruamel.yaml library
+    yaml.default_flow_style = False  # Ensures pretty YAML formatting
+
+    yaml.representer.add_representer(
+        VariableStatus,
+        lambda dumper, data: dumper.represent_scalar(
+            "tag:yaml.org,2002:str",
+            data.value,
+        ),
+    )
+
+    return yaml
+
+
+def _model_to_yaml_with_comments(
+    model_instance: CompleteResponse,
+    file_name: str,
+    start_comment: str,
+    custom_directory: Path | None = None,
+) -> Path:
+    """Convert a model instance to a structured YAML file with Norwegian descriptions as comments.
+
+    Adds Norwegian descriptions to the model, organizes fields into sections, and saves
+    the YAML file with a structured format and timestamped filename.
+
+    Args:
+        model_instance: The model instance to convert.
+        file_name: Name of the generated YAML file.
+        start_comment: Comment at the top of the file.
+        custom_directory: Optional directory to save the file.
+
+    Returns:
+        Path: The file path of the generated YAML file.
+    """
+    yaml = _configure_yaml()
+
+    from dapla_metadata.variable_definitions.variable_definition import (
+        VariableDefinition,
+    )
+
+    # Apply new fields to model
+    apply_norwegian_descriptions_to_model(VariableDefinition)
+
+    # Convert Pydantic model instance to dictionary
+    data = model_instance.model_dump(
+        serialize_as_any=True,
+        warnings="error",
+    )
+
+    # One CommentMap for each section in the yaml file
+    machine_generated_map = CommentedMap()
+    commented_map = CommentedMap()
+    status_map = CommentedMap()
+    owner_map = CommentedMap()
+
+    # Loop through all fields in the model and populate the commented maps
+    for field_name, value in data.items():
+        if field_name == VARIABLE_STATUS_FIELD_NAME:
+            _populate_commented_map(field_name, value, status_map, model_instance)
+        elif field_name == OWNER_FIELD_NAME:
+            _populate_commented_map(field_name, value, owner_map, model_instance)
+        elif field_name in MACHINE_GENERATED_FIELDS:
+            _populate_commented_map(
+                field_name,
+                value,
+                machine_generated_map,
+                model_instance,
+            )
+        elif field_name not in {VARIABLE_STATUS_FIELD_NAME, OWNER_FIELD_NAME}:
+            _populate_commented_map(field_name, value, commented_map, model_instance)
+
+    base_path = (
+        _get_variable_definitions_dir()
+        if custom_directory is None
+        else _validate_and_create_directory(custom_directory)
+    )
+
+    file_path = base_path / file_name
+
+    # It is important to preserve the order of the yaml dump operations when writing to file
+    # so that the file is predictable for the user
+    with file_path.open("w", encoding="utf-8") as file:
+        commented_map.yaml_set_start_comment(start_comment)
+        yaml.dump(commented_map, file)
+
+        status_map.yaml_set_start_comment(TEMPLATE_SECTION_HEADER_STATUS)
+        yaml.dump(status_map, file)
+
+        owner_map.yaml_set_start_comment(TEMPLATE_SECTION_HEADER_OWNER)
+        yaml.dump(owner_map, file)
+
+        machine_generated_map.yaml_set_start_comment(
+            TEMPLATE_SECTION_HEADER_MACHINE_GENERATED,
+        )
+        yaml.dump(machine_generated_map, file)
+    return file_path

dapla_metadata/variable_definitions/_utils/template_files.py

@@ -0,0 +1,112 @@
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
+    CompleteResponse,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.contact import (
+    Contact,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.language_string_type import (
+    LanguageStringType,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.owner import (
+    Owner,
+)
+from dapla_metadata.variable_definitions._generated.vardef_client.models.variable_status import (
+    VariableStatus,
+)
+from dapla_metadata.variable_definitions._utils.constants import DEFAULT_DATE
+from dapla_metadata.variable_definitions._utils.constants import TEMPLATE_HEADER
+from dapla_metadata.variable_definitions._utils.files import _create_file_name
+from dapla_metadata.variable_definitions._utils.files import _get_current_time
+from dapla_metadata.variable_definitions._utils.files import (
+    _get_variable_definitions_dir,
+)
+from dapla_metadata.variable_definitions._utils.files import (
+    _model_to_yaml_with_comments,
+)
+from dapla_metadata.variable_definitions._utils.files import logger
+
+if TYPE_CHECKING:
+    from dapla_metadata.variable_definitions.variable_definition import (
+        VariableDefinition,
+    )
+
+
+def _get_default_template() -> "VariableDefinition":
+    from dapla_metadata.variable_definitions.variable_definition import (
+        VariableDefinition,
+    )
+
+    return VariableDefinition(
+        name=LanguageStringType(
+            nb="default navn",
+            nn="default namn",
+            en="default name",
+        ),
+        short_name="default_kortnavn",
+        definition=LanguageStringType(
+            nb="default definisjon",
+            nn="default definisjon",
+            en="default definition",
+        ),
+        classification_reference="class_id",
+        valid_from=DEFAULT_DATE,
+        unit_types=["00"],
+        subject_fields=["aa"],
+        contains_special_categories_of_personal_data=False,
+        variable_status=VariableStatus.DRAFT.value,
+        owner=Owner(team="default team", groups=["default group"]),
+        contact=Contact(
+            title=LanguageStringType(
+                nb="default tittel",
+                nn="default tittel",
+                en="default title",
+            ),
+            email="default@ssb.no",
+        ),
+        id="",
+        patch_id=0,
+        created_at=DEFAULT_DATE,
+        created_by="",
+        last_updated_at=DEFAULT_DATE,
+        last_updated_by="",
+    )
+
+
+def create_template_yaml(
+    model_instance: CompleteResponse | None = None,
+    custom_directory: Path | None = None,
+) -> Path:
+    """Creates a template yaml file for a new variable definition."""
+    if model_instance is None:
+        model_instance = _get_default_template()
+    file_name = _create_file_name(
+        "variable_definition_template",
+        _get_current_time(),
+    )
+
+    file_path = _model_to_yaml_with_comments(
+        model_instance,
+        file_name,
+        TEMPLATE_HEADER,
+        custom_directory=custom_directory,
+    )
+    logger.debug("Created %s", file_path)
+    return file_path
+
+
+def _find_latest_template_file(directory: Path | None = None) -> Path | None:
+    def _filter_template_file(path: Path) -> bool:
+        return "variable_definition_template" in path.stem and path.suffix == ".yaml"
+
+    try:
+        return sorted(
+            filter(
+                _filter_template_file,
+                (directory or _get_variable_definitions_dir()).iterdir(),
+            ),
+        )[-1]
+    except IndexError:
+        return None

dapla_metadata/variable_definitions/_utils/variable_definition_files.py

@@ -0,0 +1,93 @@
+"""Utilities for writing and reading existing variable definition files."""
+
+import logging
+from os import PathLike
+from pathlib import Path
+from typing import TypeVar
+
+from pydantic import BaseModel
+from ruamel.yaml import YAML
+
+from dapla_metadata.variable_definitions._generated.vardef_client.models.complete_response import (
+    CompleteResponse,
+)
+from dapla_metadata.variable_definitions._utils.constants import HEADER
+from dapla_metadata.variable_definitions._utils.files import _create_file_name
+from dapla_metadata.variable_definitions._utils.files import _get_current_time
+from dapla_metadata.variable_definitions._utils.files import (
+    _model_to_yaml_with_comments,
+)
+
+logger = logging.getLogger(__name__)
+
+
+T = TypeVar("T", bound=BaseModel)
+
+
+def create_variable_yaml(
+    model_instance: CompleteResponse,
+    custom_directory: Path | None = None,
+) -> Path:
+    """Creates a yaml file for an existing variable definition."""
+    file_name = _create_file_name(
+        "variable_definition",
+        _get_current_time(),
+        model_instance.short_name,
+        model_instance.id,
+    )
+
+    return _model_to_yaml_with_comments(
+        model_instance,
+        file_name,
+        HEADER,
+        custom_directory=custom_directory,
+    )
+
+
+def _read_variable_definition_file(file_path: Path) -> dict:
+    yaml = YAML()
+
+    logger.debug("Full path to variable definition file %s", file_path)
+    logger.info("Reading from '%s'", file_path.name)
+    with file_path.open(encoding="utf-8") as f:
+        return yaml.load(f)
+
+
+def _read_file_to_model(
+    file_path: PathLike[str] | None,
+    model_class: type[T],
+) -> T:
+    """Read from a variable definition file into the given Pydantic model.
+
+    Args:
+        file_path (PathLike[str]): The path to the file to read in.
+        model_class (type[T]): The model to instantiate. Must inherit from Pydantic's BaseModel.
+
+    Raises:
+        TypeError: If no file path could be deduced.
+        FileNotFoundError: If we could not instantiate the model.
+
+    Returns:
+        T: BaseModel: The instantiated Pydantic model
+    """
+    try:
+        file_path = Path(
+            # type incongruence (i.e. None) is handled by catching the exception
+            file_path,  # type: ignore [arg-type]
+        )
+    except TypeError as e:
+        msg = "Could not deduce a path to the file. Please supply a path to the yaml file you wish to submit with the `file_path` parameter."
+        raise FileNotFoundError(
+            msg,
+        ) from e
+    model = model_class.from_dict(  # type:ignore [attr-defined]
+        _read_variable_definition_file(
+            file_path,
+        ),
+    )
+
+    if model is None:
+        msg = f"Could not read data from {file_path}"
+        raise FileNotFoundError(msg)
+
+    return model

dapla_metadata/variable_definitions/exceptions.py

@@ -2,13 +2,35 @@
 
 import json
 from functools import wraps
+from http import HTTPStatus
+from types import MappingProxyType
 
-from dapla_metadata.variable_definitions.generated.vardef_client.exceptions import (
+import urllib3
+from pytz import UnknownTimeZoneError
+from ruamel.yaml.error import YAMLError
+
+from dapla_metadata.variable_definitions._generated.vardef_client.exceptions import (
     OpenApiException,
 )
+from dapla_metadata.variable_definitions._generated.vardef_client.exceptions import (
+    UnauthorizedException,
+)
+
+# Use MappingProxyType so the dict is immutable
+STATUS_EXPLANATIONS: MappingProxyType[HTTPStatus | None, str] = MappingProxyType(
+    {
+        HTTPStatus.BAD_REQUEST: "There was a problem with the supplied data. Please review the data you supplied based on the details below.",
+        HTTPStatus.UNAUTHORIZED: "There is a problem with your token, it may have expired. Try logging out, logging in and trying again.",
+        HTTPStatus.FORBIDDEN: "Forbidden. You don't have access to this, possibly because your team is not an owner of this variable definition.",
+        HTTPStatus.NOT_FOUND: "The variable definition was not found, check that the `short_name` or `id` is correct and that the variable exists.",
+        HTTPStatus.METHOD_NOT_ALLOWED: "That won't work here. The status of your variable definition likely doesn't allow for this to happen.",
+        HTTPStatus.CONFLICT: "There was a conflict with existing data. Please change your data and try again.",
+        None: "",
+    },
+)
 
 
-class VardefClientException(OpenApiException):
+class VardefClientError(Exception):
     """Custom exception to represent errors encountered in the Vardef client.
 
     This exception extracts and formats error details from a JSON response body
@@ -33,24 +55,37 @@ class VardefClientException(OpenApiException):
             response_body (str): The raw response body string, stored for
                                 debugging purposes.
         """
+        self.status: int | None = None
+        self.detail: str = ""
         try:
             data = json.loads(response_body)
-            self.status = data.get("status", "Unknown status")
+            self.status = data.get("status")
             if data.get("title") == "Constraint Violation":
                 violations = data.get("violations", [])
-                self.detail = "".join(
-                    f"\n{violation.get('field', 'Unknown field')}: {violation.get('message', 'No message provided')}"
+                self.detail = "\n" + "\n".join(
+                    f"{violation.get('field', 'Unknown field')}: {violation.get('message', 'No message provided')}"
                     for violation in violations
                 )
-
             else:
-                self.detail = data.get("detail", "No detail provided")
+                self.detail = data.get("detail")
+
+            if self.detail:
+                self.detail = f"\nDetail: {self.detail}"
             self.response_body = response_body
         except (json.JSONDecodeError, TypeError):
-            self.status = "Unknown"
             self.detail = "Could not decode error response from API"
-            data = None
-        super().__init__(f"Status {self.status}: {self.detail}")
+
+        super().__init__(
+            f"{self._get_status_explanation(self.status)}{self.detail if self.detail else ''}",
+        )
+
+    @staticmethod
+    def _get_status_explanation(status: int | None) -> str:
+        return (
+            ""
+            if not status
+            else (STATUS_EXPLANATIONS.get(HTTPStatus(status)) or f"Status {status}:")
+        )
 
 
 def vardef_exception_handler(method):  # noqa: ANN201, ANN001
@@ -60,7 +95,102 @@ def vardef_exception_handler(method):  # noqa: ANN201, ANN001
     def _impl(self, *method_args, **method_kwargs):  # noqa: ANN001, ANN002, ANN003
         try:
             return method(self, *method_args, **method_kwargs)
+        except urllib3.exceptions.HTTPError as e:
+            # Catch all urllib3 exceptions by catching the base class.
+            # These exceptions typically arise from lower level network problems.
+            raise VardefClientError(
+                json.dumps(
+                    {
+                        "status": None,
+                        "title": "Network problems",
+                        "detail": f"""There was a network problem when sending the request to the server. Try again shortly.
+Original exception:
+{getattr(e, "message", repr(e))}""",
+                    },
+                ),
+            ) from e
+        except UnauthorizedException as e:
+            raise VardefClientError(
+                json.dumps(
+                    {
+                        "status": e.status,
+                        "title": "Unauthorized",
+                        "detail": "Unauthorized",
+                    },
+                ),
+            ) from e
         except OpenApiException as e:
-            raise VardefClientException(e.body) from e
+            raise VardefClientError(e.body) from e
+
+    return _impl
+
+
+class VariableNotFoundError(Exception):
+    """Custom exception for when a variable is not found.
+
+    Attributes:
+        message (str): Message describing the error.
+    """
+
+    def __init__(self, message: str) -> None:
+        """Initialize the VariableNotFoundError.
+
+        Args:
+            message (str): Description of the error.
+        """
+        super().__init__(message)
+        self.message = message
+
+    def __str__(self) -> str:
+        """Return the string representation of the exception."""
+        return f" {self.message}"
+
+
+class VardefFileError(Exception):
+    """Custom exception for catching errors related to variable definition file handling.
+
+    Attributes:
+        message (str): Message describing the error.
+    """
+
+    def __init__(self, message: str, *args) -> None:  # noqa: ANN002
+        """Accepting the message and any additional arguments."""
+        super().__init__(message, *args)
+        self.message = message
+        self.args = args
+
+    def __str__(self) -> str:
+        """Returning a custom string representation of the exception."""
+        return f"VardefFileError: {self.message}"
+
+
+def vardef_file_error_handler(method):  # noqa: ANN201, ANN001
+    """Decorator for handling exceptions when generating yaml files for variable definitions."""
+
+    @wraps(method)
+    def _impl(*method_args, **method_kwargs):  # noqa: ANN002, ANN003
+        try:
+            return method(
+                *method_args,
+                **method_kwargs,
+            )
+        except FileExistsError as e:
+            msg = f"File already exists and can not be saved: {method_kwargs.get('file_path', 'unknown file path')}"
+            raise VardefFileError(msg) from e
+        except PermissionError as e:
+            msg = f"Permission denied for file path when accessing the file. Original error: {e!s}"
+            raise VardefFileError(msg) from e
+        except UnknownTimeZoneError as e:
+            msg = f"Timezone is unknown: {method_kwargs.get('time_zone', 'unknown')}"
+            raise VardefFileError(msg) from e
+        except YAMLError as e:
+            msg = f"Invalid yaml. Please fix the formatting in your yaml file.\nOriginal error:\n{e!s}"
+            raise VardefFileError(msg) from e
+        except EOFError as e:
+            msg = "Unexpected end of file"
+            raise VardefFileError(msg) from e
+        except NotADirectoryError as e:
+            msg = f"Path is not a directory: {method_kwargs.get('file_path', 'unknown file path')}. Original error: {e!s}"
+            raise VardefFileError(msg) from e
 
     return _impl