dapla-toolbelt-metadata 0.2.1__py3-none-any.whl → 0.9.11__py3-none-any.whl

dapla_metadata/__init__.py

@@ -1,5 +1,15 @@
 """Tools and clients for working with the Dapla Metadata system."""
 
-import datadoc_model.model as datadoc_model
+import warnings
 
+warnings.filterwarnings(
+    "ignore",
+    message="As the c extension couldn't be imported, `google-crc32c` is using a pure python implementation that is significantly slower.",
+)
+
+import datadoc_model.all_optional.model as datadoc_model
+
+from . import dapla
 from . import datasets
+from . import standards
+from . import variable_definitions

dapla_metadata/_shared/__init__.py

@@ -0,0 +1 @@
+"""Utility code intended to be used by multiple packages within this project, but not by end users."""

dapla_metadata/_shared/config.py

@@ -0,0 +1,109 @@
+"""Configuration management for dataset package."""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from pprint import pformat
+
+from dotenv import dotenv_values
+from dotenv import load_dotenv
+
+from dapla_metadata._shared.enums import DaplaEnvironment
+from dapla_metadata._shared.enums import DaplaRegion
+from dapla_metadata._shared.enums import DaplaService
+
+logger = logging.getLogger(__name__)
+
+DOT_ENV_FILE_PATH = Path(__file__).parent.joinpath(".env")
+
+JUPYTERHUB_USER = "JUPYTERHUB_USER"
+DAPLA_REGION = "DAPLA_REGION"
+DAPLA_ENVIRONMENT = "DAPLA_ENVIRONMENT"
+DAPLA_SERVICE = "DAPLA_SERVICE"
+DAPLA_GROUP_CONTEXT = "DAPLA_GROUP_CONTEXT"
+OIDC_TOKEN = "OIDC_TOKEN"  # noqa: S105
+
+
+DATADOC_STATISTICAL_SUBJECT_SOURCE_URL_DEFAULT = (
+    "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
+)
+
+
+env_loaded = False
+
+
+def _load_dotenv_file() -> None:
+    global env_loaded  # noqa: PLW0603
+    if not env_loaded and DOT_ENV_FILE_PATH.exists():
+        load_dotenv(DOT_ENV_FILE_PATH)
+        env_loaded = True
+        logger.info(
+            "Loaded .env file with config keys: \n%s",
+            pformat(list(dotenv_values(DOT_ENV_FILE_PATH).keys())),
+        )
+
+
+def get_config_item(item: str, *, raising: bool = False) -> str | None:
+    """Get a config item. Makes sure all access is logged.
+
+    Args:
+        item: The name of the environment variable to obtain.
+        raising: `True` if an exception should be raised when the item isn't present.
+
+    Returns:
+        The set value or `None`
+
+    Raises:
+        OSError: Only if `raising` is True and the item is not found.
+    """
+    _load_dotenv_file()
+    value = os.environ.get(item)
+    if raising and not value:
+        msg = f"Environment variable {item} not defined."
+        raise OSError(msg)
+    logger.debug("Config accessed. %s", item)
+    return value
+
+
+def get_statistical_subject_source_url() -> str | None:
+    """Get the URL to the statistical subject source."""
+    return (
+        get_config_item("DATADOC_STATISTICAL_SUBJECT_SOURCE_URL")
+        or DATADOC_STATISTICAL_SUBJECT_SOURCE_URL_DEFAULT
+    )
+
+
+def get_dapla_region() -> DaplaRegion | None:
+    """Get the Dapla region we're running on."""
+    if region := get_config_item(DAPLA_REGION):
+        return DaplaRegion(region)
+
+    return None
+
+
+def get_dapla_environment() -> DaplaEnvironment | None:
+    """Get the Dapla environment we're running on."""
+    if env := get_config_item(DAPLA_ENVIRONMENT):
+        return DaplaEnvironment(env)
+
+    return None
+
+
+def get_dapla_service() -> DaplaService | None:
+    """Get the Dapla service we're running on."""
+    if service := get_config_item(DAPLA_SERVICE):
+        return DaplaService(service)
+
+    return None
+
+
+def get_oidc_token(*, raising: bool = False) -> str | None:
+    """Get the JWT token from the environment."""
+    return get_config_item(OIDC_TOKEN, raising=raising)
+
+
+def get_dapla_group_context(*, raising: bool = False) -> str | None:
+    """Get the group which the user has chosen to represent."""
+    return get_config_item(DAPLA_GROUP_CONTEXT, raising=raising)

dapla_metadata/_shared/enums.py

@@ -0,0 +1,27 @@
+from enum import Enum
+
+
+class DaplaRegion(str, Enum):
+    """Dapla platforms/regions."""
+
+    DAPLA_LAB = "DAPLA_LAB"
+    ON_PREM = "ON_PREM"
+    CLOUD_RUN = "CLOUD_RUN"
+
+
+class DaplaEnvironment(str, Enum):
+    """Dapla lifecycle environment."""
+
+    PROD = "PROD"
+    TEST = "TEST"
+    DEV = "DEV"
+
+
+class DaplaService(str, Enum):
+    """Dapla services."""
+
+    DATADOC = "DATADOC"
+    JUPYTERLAB = "JUPYTERLAB"
+    VS_CODE = "VS_CODE"
+    R_STUDIO = "R_STUDIO"
+    KILDOMATEN = "KILDOMATEN"

dapla_metadata/dapla/__init__.py

@@ -0,0 +1,4 @@
+"""Expose information specific to the Dapla platform."""
+
+from .user_info import DaplaLabUserInfo
+from .user_info import UserInfo

dapla_metadata/dapla/user_info.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import contextlib
+import logging
+from typing import Protocol
+
+import jwt
+
+from dapla_metadata._shared import config
+from dapla_metadata._shared.enums import DaplaRegion
+
+logger = logging.getLogger(__name__)
+
+
+class UserInfo(Protocol):
+    """Information about the current user.
+
+    Implementations may be provided for different platforms or testing.
+    """
+
+    @property
+    def short_email(self) -> str | None:
+        """Get the short email address."""
+        ...
+
+    @property
+    def current_group(self) -> str:
+        """Get the group which the user is currently representing."""
+        ...
+
+    @property
+    def current_team(self) -> str:
+        """Get the team which the user is currently representing."""
+        ...
+
+
+class UnknownUserInfo:
+    """Fallback when no implementation is found."""
+
+    @property
+    def short_email(self) -> str | None:
+        """Unknown email address."""
+        return None
+
+    @property
+    def current_group(self) -> str:
+        """Get the group which the user is currently representing."""
+        return ""
+
+    @property
+    def current_team(self) -> str:
+        """Get the team which the user is currently representing."""
+        return ""
+
+
+class TestUserInfo:
+    """Information about the current user for local development and testing."""
+
+    PLACEHOLDER_EMAIL_ADDRESS = "default_user@ssb.no"
+    PLACEHOLDER_GROUP = "default-team-developers"
+    PLACEHOLDER_TEAM = "default-team"
+
+    @property
+    def short_email(self) -> str | None:
+        """Get the short email address."""
+        return TestUserInfo.PLACEHOLDER_EMAIL_ADDRESS
+
+    @property
+    def current_group(self) -> str | None:
+        """Get the group which the user is currently representing."""
+        return TestUserInfo.PLACEHOLDER_GROUP
+
+    @property
+    def current_team(self) -> str | None:
+        """Get the team which the user is currently representing."""
+        return TestUserInfo.PLACEHOLDER_TEAM
+
+
+class DaplaLabUserInfo:
+    """Information about the current user when running on Dapla Lab."""
+
+    @property
+    def short_email(self) -> str | None:
+        """Get the short email address."""
+        encoded_jwt = config.get_oidc_token()
+        if encoded_jwt:
+            # The JWT has been verified by the platform prior to injection, no need to verify.
+            decoded_jwt = jwt.decode(encoded_jwt, options={"verify_signature": False})
+            with contextlib.suppress(KeyError):
+                # If email can't be found in the JWT, fall through and return None
+                return decoded_jwt["email"]
+
+        logger.warning(
+            "Could not access JWT from environment. Could not get short email address.",
+        )
+        return None
+
+    @property
+    def current_group(self) -> str:
+        """Get the group which the user is currently representing."""
+        if group := config.get_dapla_group_context():
+            return group
+        msg = "DAPLA_GROUP_CONTEXT environment variable not found"
+        raise OSError(msg)
+
+    @property
+    def current_team(self) -> str:
+        """Get the team which the user is currently representing."""
+        return parse_team_name(self.current_group)
+
+
+def get_user_info_for_current_platform() -> UserInfo:
+    """Return the correct implementation of UserInfo for the current platform."""
+    if config.get_dapla_region() == DaplaRegion.DAPLA_LAB:
+        return DaplaLabUserInfo()
+    logger.warning(
+        "Was not possible to retrieve user information! Some fields may not be set.",
+    )
+    return UnknownUserInfo()
+
+
+def parse_team_name(group: str) -> str:
+    """Parses the group to get the current team.
+
+    >>> parse_team_name("dapla-metadata-developers")
+    'dapla-metadata'
+
+    >>> parse_team_name("dapla-metadata-data-admins")
+    'dapla-metadata'
+
+    >>> parse_team_name("dapla-metadata")
+    'dapla'
+
+    >>> parse_team_name("dapla-metadata-not-real-name")
+    'dapla-metadata-not-real'
+    """
+    parts = group.split("-")
+    return "-".join(parts[:-2] if group.endswith("data-admins") else parts[:-1])

dapla_metadata/datasets/__init__.py

@@ -1,6 +1,6 @@
 """Document dataset."""
 
-from datadoc_model import model
+from datadoc_model.all_optional import model
 
 from .core import Datadoc
 from .dapla_dataset_path_info import DaplaDatasetPathInfo

dapla_metadata/datasets/_merge.py

@@ -0,0 +1,333 @@
+"""Code relating to merging metadata from an existing metadata document and metadata extracted from a new dataset.
+
+This is primarily convenience functionality for users whereby they can programmatically generate metadata without
+having to manually enter it. This is primarily useful when data is sharded by time (i.e. each dataset applies for
+a particular period like a month or a year). Assuming there aren't structural changes, the metadata may be reused
+for all periods.
+
+It is important to be able to detect changes in the structure of the data and warn users about this so that they can
+make changes as appropriate.
+"""
+
+import copy
+import logging
+import warnings
+from collections.abc import Iterable
+from dataclasses import dataclass
+from dataclasses import field
+from pathlib import Path
+from typing import cast
+
+import datadoc_model
+import datadoc_model.all_optional.model as all_optional_model
+import datadoc_model.required.model as required_model
+from cloudpathlib import CloudPath
+
+from dapla_metadata.datasets.dapla_dataset_path_info import DaplaDatasetPathInfo
+from dapla_metadata.datasets.utility.constants import (
+    DATASET_FIELDS_FROM_EXISTING_METADATA,
+)
+from dapla_metadata.datasets.utility.constants import INCONSISTENCIES_MESSAGE
+from dapla_metadata.datasets.utility.utils import OptionalDatadocMetadataType
+from dapla_metadata.datasets.utility.utils import VariableListType
+
+logger = logging.getLogger(__name__)
+
+BUCKET_NAME_MESSAGE = "Bucket name"
+DATA_PRODUCT_NAME_MESSAGE = "Data product name"
+DATASET_STATE_MESSAGE = "Dataset state"
+DATASET_SHORT_NAME_MESSAGE = "Dataset short name"
+VARIABLES_ADDITIONAL_MESSAGE = (
+    "Dataset has additional variables than defined in metadata"
+)
+VARIABLE_RENAME_MESSAGE = "Variables have been renamed in the dataset"
+VARIABLE_ORDER_MESSAGE = "The order of variables in the dataset has changed"
+VARIABLE_DATATYPES_MESSAGE = "Variable datatypes differ"
+VARIABLES_FEWER_MESSAGE = "Dataset has fewer variables than defined in metadata"
+
+
+class InconsistentDatasetsWarning(UserWarning):
+    """Existing and new datasets differ significantly from one another."""
+
+
+class InconsistentDatasetsError(ValueError):
+    """Existing and new datasets differ significantly from one another."""
+
+
+@dataclass
+class DatasetConsistencyStatus:
+    """Store the status for different aspects of dataset consistency.
+
+    Attributes:
+        message: Communicates to the user what aspect is inconsistent.
+        success: False if inconsistency is detected.
+        variables: Optionally communicate which variables are affected.
+    """
+
+    message: str
+    success: bool
+    variables: Iterable[str] = field(default_factory=list)
+
+    def __str__(self) -> str:
+        """Format the user message."""
+        message = self.message
+        if self.variables:
+            message += f"\n\tVariables: {self.variables}"
+        return message
+
+
+def check_dataset_consistency(
+    new_dataset_path: Path | CloudPath,
+    existing_dataset_path: Path | CloudPath,
+) -> list[DatasetConsistencyStatus]:
+    """Run consistency tests.
+
+    Args:
+        new_dataset_path: Path to the dataset to be documented.
+        existing_dataset_path: Path stored in the existing metadata.
+
+    Returns:
+        List of consistency check results.
+    """
+    new_dataset_path_info = DaplaDatasetPathInfo(new_dataset_path)
+    existing_dataset_path_info = DaplaDatasetPathInfo(existing_dataset_path)
+    return [
+        DatasetConsistencyStatus(
+            message=BUCKET_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.bucket_name
+                == existing_dataset_path_info.bucket_name
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATA_PRODUCT_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.statistic_short_name
+                == existing_dataset_path_info.statistic_short_name
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATASET_STATE_MESSAGE,
+            success=(
+                new_dataset_path_info.dataset_state
+                == existing_dataset_path_info.dataset_state
+            ),
+        ),
+        DatasetConsistencyStatus(
+            message=DATASET_SHORT_NAME_MESSAGE,
+            success=(
+                new_dataset_path_info.dataset_short_name
+                == existing_dataset_path_info.dataset_short_name
+            ),
+        ),
+    ]
+
+
+def check_variables_consistency(
+    extracted_variables: VariableListType,
+    existing_variables: VariableListType,
+) -> list[DatasetConsistencyStatus]:
+    """Check for consistency in variables structure.
+
+    Compares the existing metadata and that extracted from the new dataset and provides
+    highly detailed feedback on what is different between them.
+
+    We don't return all the results because that could create conflicting messages and false positives.
+
+    Args:
+        extracted_variables (VariableListType): Variables extracted from the new dataset.
+        existing_variables (VariableListType): Variables already documented in existing metadata
+
+    Returns:
+        list[DatasetConsistencyStatus]: The list of checks and whether they were successful.
+    """
+    extracted_names_set = {v.short_name or "" for v in extracted_variables}
+    existing_names_set = {v.short_name or "" for v in existing_variables}
+    same_length = len(extracted_variables) == len(existing_variables)
+    more_extracted_variables = extracted_names_set.difference(existing_names_set)
+    fewer_extracted_variables = existing_names_set.difference(extracted_names_set)
+    results = []
+    if same_length:
+        if more_extracted_variables:
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_RENAME_MESSAGE,
+                    variables=more_extracted_variables,
+                    success=not bool(more_extracted_variables),
+                )
+            )
+        else:
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_ORDER_MESSAGE,
+                    success=[v.short_name or "" for v in extracted_variables]
+                    == [v.short_name or "" for v in existing_variables],
+                )
+            )
+            results.append(
+                DatasetConsistencyStatus(
+                    message=VARIABLE_DATATYPES_MESSAGE,
+                    success=[v.data_type for v in extracted_variables]
+                    == [v.data_type for v in existing_variables],
+                )
+            )
+    else:
+        results.extend(
+            [
+                DatasetConsistencyStatus(
+                    message=VARIABLES_ADDITIONAL_MESSAGE,
+                    variables=more_extracted_variables,
+                    success=not bool(more_extracted_variables),
+                ),
+                DatasetConsistencyStatus(
+                    message=VARIABLES_FEWER_MESSAGE,
+                    variables=fewer_extracted_variables,
+                    success=not bool(fewer_extracted_variables),
+                ),
+            ]
+        )
+    return results
+
+
+def check_ready_to_merge(
+    results: list[DatasetConsistencyStatus], *, errors_as_warnings: bool
+) -> None:
+    """Check if the datasets are consistent enough to make a successful merge of metadata.
+
+    Args:
+        results: List if dict with property name and boolean success flag
+        errors_as_warnings: True if failing checks should be raised as warnings, not errors.
+
+    Raises:
+        InconsistentDatasetsError: If inconsistencies are found and `errors_as_warnings == False`
+    """
+    if failures := [result for result in results if not result.success]:
+        messages_list = "\n - ".join(str(f) for f in failures)
+        msg = f"{INCONSISTENCIES_MESSAGE}\n - {messages_list}"
+        if errors_as_warnings:
+            warnings.warn(
+                message=msg,
+                category=InconsistentDatasetsWarning,
+                stacklevel=2,
+            )
+        else:
+            raise InconsistentDatasetsError(
+                msg,
+            )
+
+
+def override_dataset_fields(
+    merged_metadata: all_optional_model.DatadocMetadata,
+    existing_metadata: all_optional_model.DatadocMetadata
+    | required_model.DatadocMetadata,
+) -> None:
+    """Overrides specific fields in the dataset of `merged_metadata` with values from the dataset of `existing_metadata`.
+
+    This function iterates over a predefined list of fields, `DATASET_FIELDS_FROM_EXISTING_METADATA`,
+    and sets the corresponding fields in the `merged_metadata.dataset` object to the values
+    from the `existing_metadata.dataset` object.
+
+    Args:
+        merged_metadata: An instance of `DatadocMetadata` containing the dataset to be updated.
+        existing_metadata: An instance of `DatadocMetadata` containing the dataset whose values are used to update `merged_metadata.dataset`.
+
+    Returns:
+        `None`.
+    """
+    if merged_metadata.dataset and existing_metadata.dataset:
+        # Override the fields as defined
+        for field in DATASET_FIELDS_FROM_EXISTING_METADATA:
+            setattr(
+                merged_metadata.dataset,
+                field,
+                getattr(existing_metadata.dataset, field),
+            )
+
+
+def merge_variables(
+    existing_metadata: OptionalDatadocMetadataType,
+    extracted_metadata: all_optional_model.DatadocMetadata,
+    merged_metadata: all_optional_model.DatadocMetadata,
+) -> all_optional_model.DatadocMetadata:
+    """Merges variables from the extracted metadata into the existing metadata and updates the merged metadata.
+
+    This function compares the variables from `extracted_metadata` with those in `existing_metadata`.
+    For each variable in `extracted_metadata`, it checks if a variable with the same `short_name` exists
+    in `existing_metadata`. If a match is found, it updates the existing variable with information from
+    `extracted_metadata`. If no match is found, the variable from `extracted_metadata` is directly added to `merged_metadata`.
+
+    Args:
+        existing_metadata: The metadata object containing the current state of variables.
+        extracted_metadata: The metadata object containing new or updated variables to merge.
+        merged_metadata: The metadata object that will contain the result of the merge.
+
+    Returns:
+        all_optional_model.DatadocMetadata: The `merged_metadata` object containing variables from both `existing_metadata`
+        and `extracted_metadata`.
+    """
+    if (
+        existing_metadata is not None
+        and existing_metadata.variables is not None
+        and extracted_metadata is not None
+        and extracted_metadata.variables is not None
+        and merged_metadata.variables is not None
+    ):
+        for extracted in extracted_metadata.variables:
+            existing = next(
+                (
+                    existing
+                    for existing in existing_metadata.variables
+                    if existing.short_name == extracted.short_name
+                ),
+                None,
+            )
+            if existing:
+                existing.id = (
+                    None  # Set to None so that it will be set assigned a fresh ID later
+                )
+                existing.contains_data_from = (
+                    extracted.contains_data_from or existing.contains_data_from
+                )
+                existing.contains_data_until = (
+                    extracted.contains_data_until or existing.contains_data_until
+                )
+                merged_metadata.variables.append(
+                    cast("datadoc_model.all_optional.model.Variable", existing)
+                )
+            else:
+                # If there is no existing metadata for this variable, we just use what we have extracted
+                merged_metadata.variables.append(extracted)
+    return merged_metadata
+
+
+def merge_metadata(
+    extracted_metadata: all_optional_model.DatadocMetadata | None,
+    existing_metadata: OptionalDatadocMetadataType,
+) -> all_optional_model.DatadocMetadata:
+    if not existing_metadata:
+        logger.warning(
+            "No existing metadata found, no merge to perform. Continuing with extracted metadata.",
+        )
+        return extracted_metadata or all_optional_model.DatadocMetadata()
+
+    if not extracted_metadata:
+        return cast("all_optional_model.DatadocMetadata", existing_metadata)
+
+    # Use the extracted metadata as a base
+    merged_metadata = all_optional_model.DatadocMetadata(
+        dataset=copy.deepcopy(extracted_metadata.dataset),
+        variables=[],
+    )
+
+    override_dataset_fields(
+        merged_metadata=merged_metadata,
+        existing_metadata=existing_metadata,
+    )
+
+    # Merge variables.
+    # For each extracted variable, copy existing metadata into the merged metadata
+    return merge_variables(
+        existing_metadata=existing_metadata,
+        extracted_metadata=extracted_metadata,
+        merged_metadata=merged_metadata,
+    )

dapla_metadata/datasets/code_list.py

@@ -90,10 +90,6 @@ class CodeList(GetExternalSource):
                 execution of data fetching.
             classification_id: The ID of the classification to retrieve.
         """
-        self.supported_languages = [
-            SupportedLanguages.NORSK_BOKMÅL,
-            SupportedLanguages.ENGLISH,
-        ]
         self._classifications: list[CodeListItem] = []
         self.classification_id = classification_id
         self.classifications_dataframes: (
@@ -117,12 +113,15 @@ class CodeList(GetExternalSource):
             and returns None.
         """
         classifications_dataframes: dict[SupportedLanguages, pd.DataFrame] = {}
-        for i in self.supported_languages:
+        for i in [
+            SupportedLanguages.NORSK_BOKMÅL,
+            SupportedLanguages.ENGLISH,
+        ]:
             try:
                 classifications_dataframes[i] = (
                     KlassClassification(
                         str(self.classification_id),
-                        i,
+                        i.lower(),  # type: ignore [arg-type]
                     )
                     .get_codes()
                     .data

dapla_metadata/datasets/compatibility/__init__.py

@@ -0,0 +1,10 @@
+"""Model Backwards Compatibility.
+
+This package contains code for upgrading existing metadata documents to the newest version of the model.
+This is analogous to a Database Migration where the structure of the data has changed and we wish to
+retain already persisted information.
+
+"""
+
+from ._utils import is_metadata_in_container_structure
+from .model_backwards_compatibility import upgrade_metadata