dapla-toolbelt-metadata 0.2.1__py3-none-any.whl → 0.9.11__py3-none-any.whl

dapla_metadata/datasets/dapla_dataset_path_info.py

@@ -14,7 +14,7 @@ from typing import Literal
 
 import arrow
 from cloudpathlib import GSPath
-from datadoc_model.model import DataSetState
+from datadoc_model.all_optional.model import DataSetState
 
 if TYPE_CHECKING:
     import datetime
@@ -141,6 +141,9 @@ class SsbDateFormat(DateFormat):
 
             >>> SSB_BIMESTER.get_floor("2003B4")
             datetime.date(2003, 7, 1)
+
+            >>> SSB_BIMESTER.get_floor("2003-B4")
+            datetime.date(2003, 7, 1)
         """
         try:
             year = period_string[:4]
@@ -170,6 +173,9 @@ class SsbDateFormat(DateFormat):
 
             >>> SSB_HALF_YEAR.get_ceil("2024H1")
             datetime.date(2024, 6, 30)
+
+            >>> SSB_HALF_YEAR.get_ceil("2024-H1")
+            datetime.date(2024, 6, 30)
         """
         try:
             year = period_string[:4]
@@ -182,7 +188,7 @@ class SsbDateFormat(DateFormat):
 
 SSB_BIMESTER = SsbDateFormat(
     name="SSB_BIMESTER",
-    regex_pattern=r"^\d{4}[B]\d{1}$",
+    regex_pattern=r"^\d{4}-?[B]\d{1}$",
     arrow_pattern="YYYYMM",
     timeframe="month",
     ssb_dates={
@@ -215,7 +221,7 @@ SSB_BIMESTER = SsbDateFormat(
 
 SSB_QUARTERLY = SsbDateFormat(
     name="SSB_QUARTERLY",
-    regex_pattern=r"^\d{4}[Q]\d{1}$",
+    regex_pattern=r"^\d{4}-?[Q]\d{1}$",
     arrow_pattern="YYYYMM",
     timeframe="month",
     ssb_dates={
@@ -240,7 +246,7 @@ SSB_QUARTERLY = SsbDateFormat(
 
 SSB_TRIANNUAL = SsbDateFormat(
     name="SSB_TRIANNUAL",
-    regex_pattern=r"^\d{4}[T]\d{1}$",
+    regex_pattern=r"^\d{4}-?[T]\d{1}$",
     arrow_pattern="YYYYMM",
     timeframe="month",
     ssb_dates={
@@ -260,7 +266,7 @@ SSB_TRIANNUAL = SsbDateFormat(
 )
 SSB_HALF_YEAR = SsbDateFormat(
     name="SSB_HALF_YEAR",
-    regex_pattern=r"^\d{4}[H]\d{1}$",
+    regex_pattern=r"^\d{4}-?[H]\d{1}$",
     arrow_pattern="YYYYMM",
     timeframe="month",
     ssb_dates={
@@ -413,6 +419,9 @@ class DaplaDatasetPathInfo:
             >>> DaplaDatasetPathInfo._extract_period_strings(['p1990Q1', 'kommune', 'v1'])
             ['1990Q1']
 
+            >>> DaplaDatasetPathInfo._extract_period_strings(['p1990-Q1', 'kommune', 'v1'])
+            ['1990-Q1']
+
             >>> DaplaDatasetPathInfo._extract_period_strings(['varehandel','v1'])
             []
         """
@@ -469,7 +478,7 @@ class DaplaDatasetPathInfo:
         """Extract the bucket name from the dataset path.
 
         Returns:
-            The bucket name or None if the dataset path is not a GCS path.
+            The bucket name or None if the dataset path is not a GCS path nor ssb bucketeer path.
 
         Examples:
             >>> DaplaDatasetPathInfo('gs://ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
@@ -483,17 +492,35 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
             None
+
+            >>> DaplaDatasetPathInfo('ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
+            None
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-data-delt
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-data-delt
+
+            >>> DaplaDatasetPathInfo('home/work/buckets/ssb-staging-dapla-felles-produkt/stat/utdata/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-produkt
         """
         prefix: str | None = None
-        if self.dataset_string.startswith(GSPath.cloud_prefix):
+        dataset_string = str(self.dataset_string)
+        if GSPath.cloud_prefix in self.dataset_string:
             prefix = GSPath.cloud_prefix
-        elif self.dataset_string.startswith(GS_PREFIX_FROM_PATHLIB):
+            _, bucket_and_rest = dataset_string.split(prefix, 1)
+        elif GS_PREFIX_FROM_PATHLIB in self.dataset_string:
             prefix = GS_PREFIX_FROM_PATHLIB
+            _, bucket_and_rest = self.dataset_string.split(prefix, 1)
+        elif "buckets/" in self.dataset_string:
+            prefix = "buckets/"
+            _, bucket_and_rest = self.dataset_string.split(prefix, 1)
         else:
             return None
 
         return pathlib.Path(
-            self.dataset_string.removeprefix(prefix),
+            bucket_and_rest,
         ).parts[0]
 
     @property
@@ -519,6 +546,15 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('my_data/simple_dataset_name.parquet').dataset_short_name
             simple_dataset_name
+
+            >>> DaplaDatasetPathInfo('gs:/ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').dataset_short_name
+            person_data
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/folk_data_p2021_v2.parquet').dataset_short_name
+            folk_data
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/dapla/bus_p2021_v2.parquet').dataset_short_name
+            bus
         """
         if self.contains_data_from or self.contains_data_until:
             short_name_sections = self.dataset_name_sections[
@@ -586,9 +622,15 @@ class DaplaDatasetPathInfo:
             >>> DaplaDatasetPathInfo('klargjorte_data/person_data_v1.parquet').dataset_state
             <DataSetState.PROCESSED_DATA: 'PROCESSED_DATA'>
 
+            >>> DaplaDatasetPathInfo('klargjorte-data/person_data_v1.parquet').dataset_state
+            <DataSetState.PROCESSED_DATA: 'PROCESSED_DATA'>
+
             >>> DaplaDatasetPathInfo('utdata/min_statistikk/person_data_v1.parquet').dataset_state
             <DataSetState.OUTPUT_DATA: 'OUTPUT_DATA'>
 
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data_v1.parquet').dataset_state
+            <DataSetState.INPUT_DATA: 'INPUT_DATA'>
+
             >>> DaplaDatasetPathInfo('my_special_data/person_data_v1.parquet').dataset_state
             None
         """
@@ -620,6 +662,12 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('person_data.parquet').dataset_version
             None
+
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data_v1.parquet').dataset_version
+            '1'
+
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data.parquet').dataset_version
+            None
         """
         minimum_elements_in_file_name: Final[int] = 2
         minimum_characters_in_version_string: Final[int] = 2
@@ -633,13 +681,37 @@ class DaplaDatasetPathInfo:
                 return last_filename_element[1:]
         return None
 
+    def _get_left_parts(
+        self,
+        dataset_path_parts: list[str],
+        state_index: int,
+    ) -> list[str]:
+        """Retrieve the path parts before the dataset state, considering bucket prefixes."""
+        bucket_prefix = {"gs:", "buckets"}
+        left_parts = dataset_path_parts[:state_index]
+
+        # Stop checking beyond the bucket prefix
+        prefix_intersection = bucket_prefix & set(left_parts)
+        if prefix_intersection:
+            first_prefix = min(
+                left_parts.index(prefix) for prefix in prefix_intersection
+            )
+            left_parts = left_parts[first_prefix:]
+
+        return (
+            []
+            if left_parts == ["/"]
+            or (left_parts[0] in bucket_prefix and len(left_parts) <= 2)
+            else left_parts
+        )
+
     @property
     def statistic_short_name(
         self,
     ) -> str | None:
         """Extract the statistical short name from the filepath.
 
-        Extract the statistical short name from the filepath right before the
+        Extract the statistical short name from the filepath either after bucket name or right before the
         dataset state based on the Dapla filepath naming convention.
 
         Returns:
@@ -650,21 +722,75 @@ class DaplaDatasetPathInfo:
             >>> DaplaDatasetPathInfo('prosjekt/befolkning/klargjorte_data/person_data_v1.parquet').statistic_short_name
             befolkning
 
+            >>> DaplaDatasetPathInfo('buckets/prosjekt/befolkning/person_data_v1.parquet').statistic_short_name
+            befolkning
+
             >>> DaplaDatasetPathInfo('befolkning/inndata/person_data_v1.parquet').statistic_short_name
             befolkning
 
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data.parquet').statistic_short_name
+            stat_name
+
+            >>> DaplaDatasetPathInfo('buckets/stat_name/utdata/person_data.parquet').statistic_short_name
+            None
+
             >>> DaplaDatasetPathInfo('befolkning/person_data.parquet').statistic_short_name
             None
+
+            >>> DaplaDatasetPathInfo('buckets/produkt/befolkning/utdata/person_data.parquet').statistic_short_name
+            befolkning
+
+            >>> DaplaDatasetPathInfo('resources/buckets/produkt/befolkning/utdata/person_data.parquet').statistic_short_name
+            befolkning
+
+            >>> DaplaDatasetPathInfo('gs://statistikk/produkt/klargjorte-data/persondata_p1990-Q1_p2023-Q4_v1/aar=2019/data.parquet').statistic_short_name
+            produkt
+
+            >>> DaplaDatasetPathInfo('gs://statistikk/produkt/persondata_p1990-Q1_p2023-Q4_v1/aar=2019/data.parquet').statistic_short_name
+            None
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/person_data_p2021_v2.parquet').statistic_short_name
+            None
         """
-        dataset_state = self.dataset_state
-        if dataset_state is not None:
-            dataset_state_names = self._extract_norwegian_dataset_state_path_part(
-                dataset_state,
-            )
-            dataset_path_parts = list(self.dataset_path.parts)
-            for i in dataset_state_names:
-                if i in dataset_path_parts and dataset_path_parts.index(i) != 0:
-                    return dataset_path_parts[dataset_path_parts.index(i) - 1]
+        if not self.dataset_state:
+            if self.bucket_name:
+                parts = self.dataset_path.parent.parts
+
+                if self.bucket_name not in parts:
+                    return None
+
+                # Find the index of bucket_name in the path
+                bucket_name_index = self.dataset_path.parent.parts.index(
+                    self.bucket_name,
+                )
+
+                # If there are parts after bucket_name, return the part immediately after it
+                if len(self.dataset_path.parent.parts) > bucket_name_index + 1:
+                    return self.dataset_path.parent.parts[bucket_name_index + 1]
+
+            return None
+
+        dataset_state_names = self._extract_norwegian_dataset_state_path_part(
+            self.dataset_state,
+        )
+        dataset_path_parts = list(self.dataset_path.parts)
+
+        for state in dataset_state_names:
+            if state not in dataset_path_parts:
+                continue
+
+            index = dataset_path_parts.index(state)
+
+            if index == 0:
+                continue
+
+            left_parts = self._get_left_parts(dataset_path_parts, index)
+
+            if not left_parts:
+                return None
+
+            return dataset_path_parts[index - 1]
+
         return None
 
     def path_complies_with_naming_standard(self) -> bool:

dapla_metadata/datasets/dataset_parser.py

@@ -5,18 +5,17 @@ Handles reading in the data and transforming data types to generic metadata type
 
 from __future__ import annotations
 
-import pathlib  # noqa: TCH003 import is needed for docs build
+import pathlib  # noqa: TC003 import is needed for docs build
 import re
-import typing as t
 from abc import ABC
 from abc import abstractmethod
 from typing import TYPE_CHECKING
 
 import pandas as pd
-from datadoc_model.model import DataType
-from datadoc_model.model import LanguageStringType
-from datadoc_model.model import LanguageStringTypeItem
-from datadoc_model.model import Variable
+from datadoc_model.all_optional.model import DataType
+from datadoc_model.all_optional.model import LanguageStringType
+from datadoc_model.all_optional.model import LanguageStringTypeItem
+from datadoc_model.all_optional.model import Variable
 from pyarrow import parquet as pq
 
 from dapla_metadata.datasets.utility.enums import SupportedLanguages
@@ -56,6 +55,8 @@ KNOWN_FLOAT_TYPES = (
 
 KNOWN_STRING_TYPES = (
     "string",
+    "string[pyarrow]",
+    "large_string",
     "str",
     "char",
     "varchar",
@@ -67,13 +68,18 @@ KNOWN_STRING_TYPES = (
 
 KNOWN_DATETIME_TYPES = (
     "timestamp",
+    "timestamp[s]",
+    "timestamp[ms]",
     "timestamp[us]",
     "timestamp[ns]",
+    "datetime",
     "datetime64",
-    " datetime64[ns]",
-    " datetime64[us]",
+    "datetime64[s]",
+    "datetime64[ms]",
+    "datetime64[us]",
+    "datetime64[ns]",
     "date",
-    "datetime",
+    "date32[day]",
     "time",
 )
 
@@ -89,9 +95,7 @@ TYPE_CORRESPONDENCE: list[tuple[tuple[str, ...], DataType]] = [
 ]
 TYPE_MAP: dict[str, DataType] = {}
 for concrete_type, abstract_type in TYPE_CORRESPONDENCE:
-    TYPE_MAP.update({c: abstract_type for c in concrete_type})
-
-TDatasetParser = t.TypeVar("TDatasetParser", bound="DatasetParser")
+    TYPE_MAP.update(dict.fromkeys(concrete_type, abstract_type))
 
 
 class DatasetParser(ABC):
@@ -112,31 +116,23 @@ class DatasetParser(ABC):
     @staticmethod
     def for_file(dataset: pathlib.Path | CloudPath) -> DatasetParser:
         """Return the correct subclass based on the given dataset file."""
-        supported_file_types: dict[
-            str,
-            type[DatasetParser],
-        ] = {
-            ".parquet": DatasetParserParquet,
-            ".sas7bdat": DatasetParserSas7Bdat,
-            ".parquet.gzip": DatasetParserParquet,
-        }
         file_type = "Unknown"
         try:
             file_type = dataset.suffix
             # Gzipped parquet files can be read with DatasetParserParquet
-            match = re.search(r"(.parquet.gzip)", str(dataset).lower())
-            file_type = ".parquet.gzip" if match else file_type
-            # Extract the appropriate reader class from the SUPPORTED_FILE_TYPES dict and return an instance of it
-            reader = supported_file_types[file_type](dataset)
+            match = re.search(PARQUET_GZIP_FILE_SUFFIX, str(dataset).lower())
+            file_type = PARQUET_GZIP_FILE_SUFFIX if match else file_type
+            # Extract the appropriate reader class from the SUPPORTED_FILE_TYPES dict
+            reader = SUPPORTED_DATASET_FILE_SUFFIXES[file_type](dataset)
         except IndexError as e:
             # Thrown when just one element is returned from split, meaning there is no file extension supplied
-            msg = f"Could not recognise file type for provided {dataset = }. Supported file types are: {', '.join(supported_file_types.keys())}"
+            msg = f"Could not recognise file type for provided {dataset = }. Supported file types are: {', '.join(SUPPORTED_DATASET_FILE_SUFFIXES.keys())}"
             raise FileNotFoundError(
                 msg,
             ) from e
         except KeyError as e:
             # In this case the file type is not supported, so we throw a helpful exception
-            msg = f"{file_type = } is not supported. Please open one of the following supported files types: {', '.join(supported_file_types.keys())} or contact the maintainers to request support."
+            msg = f"{file_type = } is not supported. Please open one of the following supported files types: {', '.join(SUPPORTED_DATASET_FILE_SUFFIXES.keys())} or contact the maintainers to request support."
             raise NotImplementedError(
                 msg,
             ) from e
@@ -157,6 +153,9 @@ class DatasetParser(ABC):
 
         Arguments:
             data_type: The concrete data type to map.
+
+        Returns:
+            The abstract data type or None
         """
         return TYPE_MAP.get(data_type.lower(), None)
 
@@ -179,11 +178,11 @@ class DatasetParserParquet(DatasetParser):
     def get_fields(self) -> list[Variable]:
         """Extract the fields from this dataset."""
         with self.dataset.open(mode="rb") as f:
-            schema: pa.Schema = pq.read_schema(f)  # type: ignore [arg-type]
+            schema: pa.Schema = pq.read_schema(f)  # type: ignore [arg-type, assignment]
         return [
             Variable(
                 short_name=data_field.name.strip(),
-                data_type=self.transform_data_type(str(data_field.type)),
+                data_type=self.transform_data_type(str(data_field.type)),  # type: ignore [attr-defined]
             )
             for data_field in schema
             if data_field.name
@@ -239,3 +238,17 @@ class DatasetParserSas7Bdat(DatasetParser):
             )
 
         return fields
+
+
+PARQUET_FILE_SUFFIX = ".parquet"
+PARQUET_GZIP_FILE_SUFFIX = ".parquet.gzip"
+SAS7BDAT_FILE_SUFFIX = ".sas7bdat"
+
+SUPPORTED_DATASET_FILE_SUFFIXES: dict[
+    str,
+    type[DatasetParser],
+] = {
+    PARQUET_FILE_SUFFIX: DatasetParserParquet,
+    PARQUET_GZIP_FILE_SUFFIX: DatasetParserParquet,
+    SAS7BDAT_FILE_SUFFIX: DatasetParserSas7Bdat,
+}

dapla_metadata/datasets/model_validation.py

@@ -13,20 +13,19 @@ from typing_extensions import Self
 
 from dapla_metadata.datasets.utility.constants import DATE_VALIDATION_MESSAGE
 from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
-from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
 from dapla_metadata.datasets.utility.constants import OBLIGATORY_METADATA_WARNING
 from dapla_metadata.datasets.utility.utils import get_missing_obligatory_dataset_fields
 from dapla_metadata.datasets.utility.utils import (
     get_missing_obligatory_variables_fields,
 )
+from dapla_metadata.datasets.utility.utils import (
+    get_missing_obligatory_variables_pseudo_fields,
+)
 from dapla_metadata.datasets.utility.utils import get_timestamp_now
 from dapla_metadata.datasets.utility.utils import incorrect_date_order
 from dapla_metadata.datasets.utility.utils import (
     num_obligatory_dataset_fields_completed,
 )
-from dapla_metadata.datasets.utility.utils import (
-    num_obligatory_variables_fields_completed,
-)
 from dapla_metadata.datasets.utility.utils import set_variables_inherit_from_dataset
 
 if TYPE_CHECKING:
@@ -146,21 +145,31 @@ class ValidateDatadocMetadata(model.DatadocMetadata):
             ObligatoryVariableWarning: If not all obligatory variable metadata fields
                 are filled in.
         """
-        if self.variables is not None and num_obligatory_variables_fields_completed(
-            self.variables,
-        ) != (NUM_OBLIGATORY_VARIABLES_FIELDS * len(self.variables)):
-            warnings.warn(
-                f"{OBLIGATORY_METADATA_WARNING} {get_missing_obligatory_variables_fields(self.variables)}",
-                ObligatoryVariableWarning,
-                stacklevel=2,
-            )
-            logger.warning(
-                "Type warning: %s.%s %s",
-                ObligatoryVariableWarning,
-                OBLIGATORY_METADATA_WARNING,
-                get_missing_obligatory_variables_fields(self.variables),
-            )
-
+        if self.variables is not None:
+            missing_fields_dict = {}
+            for d in get_missing_obligatory_variables_fields(self.variables):
+                for var, fields in d.items():
+                    missing_fields_dict[var] = fields.copy()
+
+            for d in get_missing_obligatory_variables_pseudo_fields(self.variables):
+                for var, fields in d.items():
+                    if var in missing_fields_dict:
+                        missing_fields_dict[var].extend(fields)
+                    else:
+                        missing_fields_dict[var] = fields.copy()
+
+            missing_fields = [
+                {var: fields} for var, fields in missing_fields_dict.items()
+            ]
+            if missing_fields:
+                message = f"{OBLIGATORY_METADATA_WARNING} {missing_fields}"
+                warnings.warn(message, ObligatoryVariableWarning, stacklevel=2)
+                logger.warning(
+                    "Type warning: %s.%s %s",
+                    ObligatoryVariableWarning,
+                    OBLIGATORY_METADATA_WARNING,
+                    missing_fields,
+                )
         return self
 
 
@@ -176,7 +185,7 @@ class ObligatoryVariableWarning(UserWarning):
     """Custom warning for checking obligatory metadata for variables."""
 
 
-def custom_warning_handler(  # noqa: PLR0913 remove fields causes incompatible types
+def custom_warning_handler(
     message: Warning | str,
     category: type[Warning],
     filename: str,

dapla_metadata/datasets/statistic_subject_mapping.py

@@ -140,7 +140,11 @@ class StatisticSubjectMapping(GetExternalSource):
                 SecondarySubject(
                     self._extract_titles(s.titler),
                     s["emnekode"],
-                    [statistikk["kortnavn"] for statistikk in s.find_all("Statistikk")],
+                    [
+                        statistikk["kortnavn"]
+                        for statistikk in s.find_all("Statistikk")
+                        if statistikk["isPrimaerPlassering"] == "true"
+                    ],
                 )
                 for s in p.find_all("delemne")
             ]

dapla_metadata/datasets/utility/constants.py

@@ -1,7 +1,7 @@
 """Repository for constant values in Datadoc backend."""
 
-from datadoc_model.model import LanguageStringType
-from datadoc_model.model import LanguageStringTypeItem
+from datadoc_model.all_optional.model import LanguageStringType
+from datadoc_model.all_optional.model import LanguageStringTypeItem
 
 VALIDATION_ERROR = "Validation error: "
 
@@ -9,7 +9,7 @@ DATE_VALIDATION_MESSAGE = f"{VALIDATION_ERROR}contains_data_from must be the sam
 
 OBLIGATORY_METADATA_WARNING = "Obligatory metadata is missing: "
 
-INCONSISTENCIES_MESSAGE = "Inconsistencies found between extracted and existing metadata. Inconsistencies are:"
+INCONSISTENCIES_MESSAGE = "Inconsistencies found between extracted and existing metadata! This usually means that the new dataset has a different structure and that the version number should be incremented.\nDetails:"
 
 OBLIGATORY_DATASET_METADATA_IDENTIFIERS: list = [
     "assessment",
@@ -17,12 +17,9 @@ OBLIGATORY_DATASET_METADATA_IDENTIFIERS: list = [
     "dataset_status",
     "name",
     "description",
-    "data_source",
     "population_description",
     "version",
     "version_description",
-    "unit_type",
-    "temporality_type",
     "subject_field",
     "spatial_coverage_description",
     "owner",
@@ -44,8 +41,18 @@ OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS = [
     "data_type",
     "variable_role",
     "is_personal_data",
+    "unit_type",
+    "population_description",
+    "data_source",
+    "temporality_type",
+]
+
+OBLIGATORY_VARIABLES_PSEUDONYMIZATION_IDENTIFIERS = [
+    "encryption_algorithm",
+    "encryption_key_reference",
 ]
 
+
 OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS_MULTILANGUAGE = [
     "name",
 ]
@@ -75,22 +82,22 @@ DATASET_FIELDS_FROM_EXISTING_METADATA = [
     "dataset_status",
     "name",
     "description",
-    "data_source",
     "population_description",
-    "unit_type",
-    "temporality_type",
     "subject_field",
     "keyword",
     "spatial_coverage_description",
-    "contains_personal_data",
-    "use_restriction",
-    "use_restriction_date",
+    "use_restrictions",
     "custom_type",
     "owner",
+    "version_description",
 ]
 
 METADATA_DOCUMENT_FILE_SUFFIX = "__DOC.json"
 
-DATADOC_STATISTICAL_SUBJECT_SOURCE_URL = (
-    "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
-)
+PAPIS_STABLE_IDENTIFIER_TYPE = "FREG_SNR"
+PAPIS_ENCRYPTION_KEY_REFERENCE = "papis-common-key-1"
+DAEAD_ENCRYPTION_KEY_REFERENCE = "ssb-common-key-1"
+ENCRYPTION_PARAMETER_SNAPSHOT_DATE = "snapshotDate"
+ENCRYPTION_PARAMETER_KEY_ID = "keyId"
+ENCRYPTION_PARAMETER_STRATEGY = "strategy"
+ENCRYPTION_PARAMETER_STRATEGY_SKIP = "skip"

dapla_metadata/datasets/utility/enums.py

@@ -5,31 +5,19 @@ from __future__ import annotations
 from enum import Enum
 
 
-class DaplaRegion(str, Enum):
-    """Dapla platforms/regions."""
-
-    DAPLA_LAB = "DAPLA_LAB"
-    BIP = "BIP"
-    ON_PREM = "ON_PREM"
-    CLOUD_RUN = "CLOUD_RUN"
-
-
-class DaplaService(str, Enum):
-    """Dapla services."""
-
-    DATADOC = "DATADOC"
-    JUPYTERLAB = "JUPYTERLAB"
-    VS_CODE = "VS_CODE"
-    R_STUDIO = "R_STUDIO"
-    KILDOMATEN = "KILDOMATEN"
-
-
 class SupportedLanguages(str, Enum):
     """The list of languages metadata may be recorded in.
 
     Reference: https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
     """
 
-    NORSK_BOKMÅL = "nb"
+    NORSK_BOKMÅL = "nb"  # noqa: PLC2401 the listed problems do not apply in this case
     NORSK_NYNORSK = "nn"
     ENGLISH = "en"
+
+
+class EncryptionAlgorithm(str, Enum):
+    """Encryption algorithm values for pseudonymization algoprithms offered on Dapla."""
+
+    PAPIS_ENCRYPTION_ALGORITHM = "TINK-FPE"
+    DAEAD_ENCRYPTION_ALGORITHM = "TINK-DAEAD"