dapla-toolbelt-metadata 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

dapla_metadata/__init__.py

@@ -9,5 +9,7 @@ warnings.filterwarnings(
 
 import datadoc_model.model as datadoc_model
 
+from . import dapla
 from . import datasets
+from . import standards
 from . import variable_definitions

dapla_metadata/dapla/__init__.py

@@ -1 +1,4 @@
 """Expose information specific to the Dapla platform."""
+
+from .user_info import DaplaLabUserInfo
+from .user_info import UserInfo

dapla_metadata/dapla/user_info.py

@@ -13,11 +13,6 @@ from dapla_metadata._shared.enums import DaplaService
 logger = logging.getLogger(__name__)
 
 
-PLACEHOLDER_EMAIL_ADDRESS = "default_user@ssb.no"
-PLACEHOLDER_GROUP = "default-team-developers"
-PLACEHOLDER_TEAM = "default-team"
-
-
 class UserInfo(Protocol):
     """Information about the current user.
 
@@ -62,20 +57,24 @@ class UnknownUserInfo:
 class TestUserInfo:
     """Information about the current user for local development and testing."""
 
+    PLACEHOLDER_EMAIL_ADDRESS = "default_user@ssb.no"
+    PLACEHOLDER_GROUP = "default-team-developers"
+    PLACEHOLDER_TEAM = "default-team"
+
     @property
     def short_email(self) -> str | None:
         """Get the short email address."""
-        return PLACEHOLDER_EMAIL_ADDRESS
+        return TestUserInfo.PLACEHOLDER_EMAIL_ADDRESS
 
     @property
     def current_group(self) -> str | None:
         """Get the group which the user is currently representing."""
-        return PLACEHOLDER_GROUP
+        return TestUserInfo.PLACEHOLDER_GROUP
 
     @property
     def current_team(self) -> str | None:
         """Get the team which the user is currently representing."""
-        return PLACEHOLDER_TEAM
+        return TestUserInfo.PLACEHOLDER_TEAM
 
 
 class DaplaLabUserInfo:
@@ -146,17 +145,17 @@ def get_user_info_for_current_platform() -> UserInfo:
 def parse_team_name(group: str) -> str:
     """Parses the group to get the current team.
 
-    >>> parse_team_name(dapla-metadata-developers)
-    (dapla-metadata)
+    >>> parse_team_name("dapla-metadata-developers")
+    'dapla-metadata'
 
-    >>> parse_team_name(dapla-metadata-data-admins)
-    (dapla-metadata)
+    >>> parse_team_name("dapla-metadata-data-admins")
+    'dapla-metadata'
 
-    >>> parse_team_name(dapla-metadata)
-    (dapla)
+    >>> parse_team_name("dapla-metadata")
+    'dapla'
 
-    >>> parse_team_name(dapla-metadata-not-real-name)
-    (dapla-metadata-not-real)
+    >>> parse_team_name("dapla-metadata-not-real-name")
+    'dapla-metadata-not-real'
     """
     parts = group.split("-")
     return "-".join(parts[:-2] if group.endswith("data-admins") else parts[:-1])

dapla_metadata/datasets/dapla_dataset_path_info.py

@@ -478,7 +478,7 @@ class DaplaDatasetPathInfo:
         """Extract the bucket name from the dataset path.
 
         Returns:
-            The bucket name or None if the dataset path is not a GCS path.
+            The bucket name or None if the dataset path is not a GCS path nor ssb bucketeer path.
 
         Examples:
             >>> DaplaDatasetPathInfo('gs://ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
@@ -492,17 +492,35 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
             None
+
+            >>> DaplaDatasetPathInfo('ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
+            None
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-data-delt
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-data-delt
+
+            >>> DaplaDatasetPathInfo('home/work/buckets/ssb-staging-dapla-felles-produkt/stat/utdata/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-produkt
         """
         prefix: str | None = None
-        if self.dataset_string.startswith(GSPath.cloud_prefix):
+        dataset_string = str(self.dataset_string)
+        if GSPath.cloud_prefix in self.dataset_string:
             prefix = GSPath.cloud_prefix
-        elif self.dataset_string.startswith(GS_PREFIX_FROM_PATHLIB):
+            _, bucket_and_rest = dataset_string.split(prefix, 1)
+        elif GS_PREFIX_FROM_PATHLIB in self.dataset_string:
             prefix = GS_PREFIX_FROM_PATHLIB
+            _, bucket_and_rest = self.dataset_string.split(prefix, 1)
+        elif "buckets/" in self.dataset_string:
+            prefix = "buckets/"
+            _, bucket_and_rest = self.dataset_string.split(prefix, 1)
         else:
             return None
 
         return pathlib.Path(
-            self.dataset_string.removeprefix(prefix),
+            bucket_and_rest,
         ).parts[0]
 
     @property
@@ -528,6 +546,15 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('my_data/simple_dataset_name.parquet').dataset_short_name
             simple_dataset_name
+
+            >>> DaplaDatasetPathInfo('gs:/ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').dataset_short_name
+            person_data
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/folk_data_p2021_v2.parquet').dataset_short_name
+            folk_data
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/dapla/bus_p2021_v2.parquet').dataset_short_name
+            bus
         """
         if self.contains_data_from or self.contains_data_until:
             short_name_sections = self.dataset_name_sections[
@@ -601,6 +628,9 @@ class DaplaDatasetPathInfo:
             >>> DaplaDatasetPathInfo('utdata/min_statistikk/person_data_v1.parquet').dataset_state
             <DataSetState.OUTPUT_DATA: 'OUTPUT_DATA'>
 
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data_v1.parquet').dataset_state
+            <DataSetState.INPUT_DATA: 'INPUT_DATA'>
+
             >>> DaplaDatasetPathInfo('my_special_data/person_data_v1.parquet').dataset_state
             None
         """
@@ -632,6 +662,12 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('person_data.parquet').dataset_version
             None
+
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data_v1.parquet').dataset_version
+            '1'
+
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data.parquet').dataset_version
+            None
         """
         minimum_elements_in_file_name: Final[int] = 2
         minimum_characters_in_version_string: Final[int] = 2
@@ -645,13 +681,37 @@ class DaplaDatasetPathInfo:
                 return last_filename_element[1:]
         return None
 
+    def _get_left_parts(
+        self,
+        dataset_path_parts: list[str],
+        state_index: int,
+    ) -> list[str]:
+        """Retrieve the path parts before the dataset state, considering bucket prefixes."""
+        bucket_prefix = {"gs:", "buckets"}
+        left_parts = dataset_path_parts[:state_index]
+
+        # Stop checking beyond the bucket prefix
+        prefix_intersection = bucket_prefix & set(left_parts)
+        if prefix_intersection:
+            first_prefix = min(
+                left_parts.index(prefix) for prefix in prefix_intersection
+            )
+            left_parts = left_parts[first_prefix:]
+
+        return (
+            []
+            if left_parts == ["/"]
+            or (left_parts[0] in bucket_prefix and len(left_parts) <= 2)
+            else left_parts
+        )
+
     @property
     def statistic_short_name(
         self,
     ) -> str | None:
         """Extract the statistical short name from the filepath.
 
-        Extract the statistical short name from the filepath right before the
+        Extract the statistical short name from the filepath either after bucket name or right before the
         dataset state based on the Dapla filepath naming convention.
 
         Returns:
@@ -662,21 +722,75 @@ class DaplaDatasetPathInfo:
             >>> DaplaDatasetPathInfo('prosjekt/befolkning/klargjorte_data/person_data_v1.parquet').statistic_short_name
             befolkning
 
+            >>> DaplaDatasetPathInfo('buckets/prosjekt/befolkning/person_data_v1.parquet').statistic_short_name
+            befolkning
+
             >>> DaplaDatasetPathInfo('befolkning/inndata/person_data_v1.parquet').statistic_short_name
             befolkning
 
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data.parquet').statistic_short_name
+            stat_name
+
+            >>> DaplaDatasetPathInfo('buckets/stat_name/utdata/person_data.parquet').statistic_short_name
+            None
+
             >>> DaplaDatasetPathInfo('befolkning/person_data.parquet').statistic_short_name
             None
+
+            >>> DaplaDatasetPathInfo('buckets/produkt/befolkning/utdata/person_data.parquet').statistic_short_name
+            befolkning
+
+            >>> DaplaDatasetPathInfo('resources/buckets/produkt/befolkning/utdata/person_data.parquet').statistic_short_name
+            befolkning
+
+            >>> DaplaDatasetPathInfo('gs://statistikk/produkt/klargjorte-data/persondata_p1990-Q1_p2023-Q4_v1/aar=2019/data.parquet').statistic_short_name
+            produkt
+
+            >>> DaplaDatasetPathInfo('gs://statistikk/produkt/persondata_p1990-Q1_p2023-Q4_v1/aar=2019/data.parquet').statistic_short_name
+            None
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/person_data_p2021_v2.parquet').statistic_short_name
+            None
         """
-        dataset_state = self.dataset_state
-        if dataset_state is not None:
-            dataset_state_names = self._extract_norwegian_dataset_state_path_part(
-                dataset_state,
-            )
-            dataset_path_parts = list(self.dataset_path.parts)
-            for i in dataset_state_names:
-                if i in dataset_path_parts and dataset_path_parts.index(i) != 0:
-                    return dataset_path_parts[dataset_path_parts.index(i) - 1]
+        if not self.dataset_state:
+            if self.bucket_name:
+                parts = self.dataset_path.parent.parts
+
+                if self.bucket_name not in parts:
+                    return None
+
+                # Find the index of bucket_name in the path
+                bucket_name_index = self.dataset_path.parent.parts.index(
+                    self.bucket_name,
+                )
+
+                # If there are parts after bucket_name, return the part immediately after it
+                if len(self.dataset_path.parent.parts) > bucket_name_index + 1:
+                    return self.dataset_path.parent.parts[bucket_name_index + 1]
+
+            return None
+
+        dataset_state_names = self._extract_norwegian_dataset_state_path_part(
+            self.dataset_state,
+        )
+        dataset_path_parts = list(self.dataset_path.parts)
+
+        for state in dataset_state_names:
+            if state not in dataset_path_parts:
+                continue
+
+            index = dataset_path_parts.index(state)
+
+            if index == 0:
+                continue
+
+            left_parts = self._get_left_parts(dataset_path_parts, index)
+
+            if not left_parts:
+                return None
+
+            return dataset_path_parts[index - 1]
+
         return None
 
     def path_complies_with_naming_standard(self) -> bool:

dapla_metadata/datasets/dataset_parser.py

@@ -89,7 +89,7 @@ TYPE_CORRESPONDENCE: list[tuple[tuple[str, ...], DataType]] = [
 ]
 TYPE_MAP: dict[str, DataType] = {}
 for concrete_type, abstract_type in TYPE_CORRESPONDENCE:
-    TYPE_MAP.update({c: abstract_type for c in concrete_type})
+    TYPE_MAP.update(dict.fromkeys(concrete_type, abstract_type))
 
 TDatasetParser = t.TypeVar("TDatasetParser", bound="DatasetParser")
 
@@ -112,31 +112,23 @@ class DatasetParser(ABC):
     @staticmethod
     def for_file(dataset: pathlib.Path | CloudPath) -> DatasetParser:
         """Return the correct subclass based on the given dataset file."""
-        supported_file_types: dict[
-            str,
-            type[DatasetParser],
-        ] = {
-            ".parquet": DatasetParserParquet,
-            ".sas7bdat": DatasetParserSas7Bdat,
-            ".parquet.gzip": DatasetParserParquet,
-        }
         file_type = "Unknown"
         try:
             file_type = dataset.suffix
             # Gzipped parquet files can be read with DatasetParserParquet
-            match = re.search(r"(.parquet.gzip)", str(dataset).lower())
-            file_type = ".parquet.gzip" if match else file_type
+            match = re.search(PARQUET_GZIP_FILE_SUFFIX, str(dataset).lower())
+            file_type = PARQUET_GZIP_FILE_SUFFIX if match else file_type
             # Extract the appropriate reader class from the SUPPORTED_FILE_TYPES dict and return an instance of it
-            reader = supported_file_types[file_type](dataset)
+            reader = SUPPORTED_DATASET_FILE_SUFFIXES[file_type](dataset)
         except IndexError as e:
             # Thrown when just one element is returned from split, meaning there is no file extension supplied
-            msg = f"Could not recognise file type for provided {dataset = }. Supported file types are: {', '.join(supported_file_types.keys())}"
+            msg = f"Could not recognise file type for provided {dataset = }. Supported file types are: {', '.join(SUPPORTED_DATASET_FILE_SUFFIXES.keys())}"
             raise FileNotFoundError(
                 msg,
             ) from e
         except KeyError as e:
             # In this case the file type is not supported, so we throw a helpful exception
-            msg = f"{file_type = } is not supported. Please open one of the following supported files types: {', '.join(supported_file_types.keys())} or contact the maintainers to request support."
+            msg = f"{file_type = } is not supported. Please open one of the following supported files types: {', '.join(SUPPORTED_DATASET_FILE_SUFFIXES.keys())} or contact the maintainers to request support."
             raise NotImplementedError(
                 msg,
             ) from e
@@ -239,3 +231,17 @@ class DatasetParserSas7Bdat(DatasetParser):
             )
 
         return fields
+
+
+PARQUET_FILE_SUFFIX = ".parquet"
+PARQUET_GZIP_FILE_SUFFIX = ".parquet.gzip"
+SAS7BDAT_FILE_SUFFIX = ".sas7bdat"
+
+SUPPORTED_DATASET_FILE_SUFFIXES: dict[
+    str,
+    type[DatasetParser],
+] = {
+    PARQUET_FILE_SUFFIX: DatasetParserParquet,
+    PARQUET_GZIP_FILE_SUFFIX: DatasetParserParquet,
+    SAS7BDAT_FILE_SUFFIX: DatasetParserSas7Bdat,
+}

dapla_metadata/datasets/utility/utils.py

@@ -5,6 +5,7 @@ import logging
 import pathlib
 import uuid
 
+import google.auth
 from cloudpathlib import CloudPath
 from cloudpathlib import GSClient
 from cloudpathlib import GSPath
@@ -13,7 +14,6 @@ from datadoc_model.model import Assessment
 from datadoc_model.model import DataSetState
 from datadoc_model.model import VariableRole
 
-from dapla import AuthClient
 from dapla_metadata.dapla import user_info
 from dapla_metadata.datasets.utility.constants import (
     DATASET_FIELDS_FROM_EXISTING_METADATA,
@@ -52,7 +52,7 @@ def normalize_path(path: str) -> pathlib.Path | CloudPath:
         Pathlib compatible object.
     """
     if path.startswith(GSPath.cloud_prefix):
-        client = GSClient(credentials=AuthClient.fetch_google_credentials())
+        client = GSClient(credentials=google.auth.default()[0])
         return GSPath(path, client=client)
     return pathlib.Path(path)
 

dapla_metadata/standards/__init__.py

@@ -0,0 +1,4 @@
+"""Expose information specific to validating ssb standards."""
+
+from .standard_validators import check_naming_standard
+from .standard_validators import generate_validation_report

dapla_metadata/standards/name_validator.py

@@ -0,0 +1,250 @@
+import asyncio
+import logging
+import os
+import re
+from collections.abc import AsyncGenerator
+from pathlib import Path
+
+from dapla_metadata.datasets.dapla_dataset_path_info import DaplaDatasetPathInfo
+from dapla_metadata.datasets.dataset_parser import SUPPORTED_DATASET_FILE_SUFFIXES
+from dapla_metadata.standards.utils.constants import FILE_DOES_NOT_EXIST
+from dapla_metadata.standards.utils.constants import FILE_IGNORED
+from dapla_metadata.standards.utils.constants import IGNORED_FOLDERS
+from dapla_metadata.standards.utils.constants import INVALID_SYMBOLS
+from dapla_metadata.standards.utils.constants import MISSING_DATA_STATE
+from dapla_metadata.standards.utils.constants import MISSING_DATASET_SHORT_NAME
+from dapla_metadata.standards.utils.constants import MISSING_PERIOD
+from dapla_metadata.standards.utils.constants import MISSING_SHORT_NAME
+from dapla_metadata.standards.utils.constants import MISSING_VERSION
+from dapla_metadata.standards.utils.constants import NAME_STANDARD_SUCCESS
+from dapla_metadata.standards.utils.constants import NAME_STANDARD_VIOLATION
+from dapla_metadata.standards.utils.constants import PATH_IGNORED
+from dapla_metadata.standards.utils.constants import SSB_NAMING_STANDARD_REPORT
+from dapla_metadata.standards.utils.constants import SSB_NAMING_STANDARD_REPORT_FILES
+from dapla_metadata.standards.utils.constants import (
+    SSB_NAMING_STANDARD_REPORT_RESULT_AVERAGE,
+)
+from dapla_metadata.standards.utils.constants import (
+    SSB_NAMING_STANDARD_REPORT_RESULT_BEST,
+)
+from dapla_metadata.standards.utils.constants import (
+    SSB_NAMING_STANDARD_REPORT_RESULT_GOOD,
+)
+from dapla_metadata.standards.utils.constants import (
+    SSB_NAMING_STANDARD_REPORT_RESULT_LOW,
+)
+from dapla_metadata.standards.utils.constants import (
+    SSB_NAMING_STANDARD_REPORT_RESULT_NO_SCORE,
+)
+from dapla_metadata.standards.utils.constants import SSB_NAMING_STANDARD_REPORT_SUCCESS
+from dapla_metadata.standards.utils.constants import (
+    SSB_NAMING_STANDARD_REPORT_SUCCESS_RATE,
+)
+from dapla_metadata.standards.utils.constants import (
+    SSB_NAMING_STANDARD_REPORT_VIOLATIONS,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ValidationResult:
+    """Result object for name standard validation."""
+
+    def __init__(
+        self,
+        success: bool,
+        file_path: str,
+    ) -> None:
+        """Initialize the validatation result."""
+        self.success = success
+        self.file_path = file_path
+        self.messages: list[str] = []
+        self.violations: list[str] = []
+
+    def add_message(self, message: str) -> None:
+        """Add message to list."""
+        if message not in self.messages:
+            self.messages.append(message)
+
+    def add_violation(self, violation: str) -> None:
+        """Add violation to list."""
+        if violation not in self.violations:
+            self.violations.append(violation)
+        if self.success:
+            self.success = False
+
+    def __repr__(self) -> str:
+        """Representation for debugging."""
+        return f"ValidationResult(success={self.success}, file_path={self.file_path}, messages={self.messages}, violations={self.violations})"
+
+    def to_dict(self) -> dict:
+        """Return result as a dictionary."""
+        return {
+            "success": self.success,
+            "file_path": self.file_path,
+            "messages": self.messages,
+            "violations": self.violations,
+        }
+
+
+class NamingStandardReport:
+    """Report object for name standard validation."""
+
+    def __init__(self, validation_results: list[ValidationResult]) -> None:
+        """Initialize the naming standard report."""
+        self.validation_results = validation_results
+        self.num_files_validated = len(validation_results)
+        self.num_success = len(
+            [result for result in validation_results if result.success is True],
+        )
+        self.num_failures = len(
+            [result for result in validation_results if result.success is False],
+        )
+
+    def generate_report(self) -> str:
+        """Format the report as a string."""
+        return (
+            f"{SSB_NAMING_STANDARD_REPORT}\n"
+            f"=============================\n"
+            f"{self.evaluate_result()}"
+            f"{SSB_NAMING_STANDARD_REPORT_SUCCESS_RATE}: {self.success_rate():.2f}%\n"
+            f"{SSB_NAMING_STANDARD_REPORT_FILES}: {self.num_files_validated}\n"
+            f"{SSB_NAMING_STANDARD_REPORT_SUCCESS}: {self.num_success}\n"
+            f"{SSB_NAMING_STANDARD_REPORT_VIOLATIONS}s: {self.num_failures}\n"
+        )
+
+    def success_rate(self) -> int | float | None:
+        """Calculate the success rate as a percentage.
+
+        Returns:
+            int | float | None: The success rate as a percentage, or None if
+            no files were validated.
+        """
+        if self.num_files_validated == 0:
+            return None
+        return self.num_success / self.num_files_validated * 100
+
+    def evaluate_result(self) -> str:
+        """Returns an appropriate message based on the success rate."""
+        rate = self.success_rate()
+        if rate is not None:
+            if rate == 100:
+                return SSB_NAMING_STANDARD_REPORT_RESULT_BEST
+            if 70 < rate < 100:
+                return SSB_NAMING_STANDARD_REPORT_RESULT_GOOD
+            if 40 <= rate <= 70:
+                return SSB_NAMING_STANDARD_REPORT_RESULT_AVERAGE
+            if rate < 40:
+                return SSB_NAMING_STANDARD_REPORT_RESULT_LOW
+        return SSB_NAMING_STANDARD_REPORT_RESULT_NO_SCORE
+
+
+def _has_invalid_symbols(path: os.PathLike[str]) -> bool:
+    """Return True if string contains illegal symbols.
+
+    Examples:
+        >>> _has_invalid_symbols("åregang-øre")
+        True
+
+        >>> _has_invalid_symbols("Azor89")
+        False
+
+        >>> _has_invalid_symbols("ssbÆ-dapla-example-data-produkt-prod/ledstill/oppdrag/skjema_p2018_p2020_v1")
+        True
+
+        >>> _has_invalid_symbols("ssb-dapla-example-data-produkt-prod/ledstill/oppdrag/skjema_p2018_p2020_v1")
+        False
+
+        >>> _has_invalid_symbols("ssb-dapla-example-data-produkt-prod/ledstill/inndata/skjema_p2018_p202_v1/aar=2018/data.parquet")
+        False
+    """
+    # TODO @mmwinther: The = symbol is allowed to avoid failures on subdirectories of partioned parquet datasets.
+    # DPMETA-824
+    return bool(re.search(r"[^a-zA-Z0-9\./:_\-=]", str(path).strip()))
+
+
+def _check_violations(
+    file: Path,
+) -> list[str]:
+    """Check for missing attributes and invalid symbols."""
+    path_info = DaplaDatasetPathInfo(file)
+    checks = {
+        MISSING_SHORT_NAME: path_info.statistic_short_name,
+        MISSING_DATA_STATE: path_info.dataset_state,
+        MISSING_PERIOD: path_info.contains_data_from,
+        MISSING_DATASET_SHORT_NAME: path_info.dataset_short_name,
+        MISSING_VERSION: path_info.dataset_version,
+        INVALID_SYMBOLS: not _has_invalid_symbols(file),
+    }
+
+    return [message for message, value in checks.items() if not value]
+
+
+async def _validate_file(
+    file: Path,
+    check_file_exists: bool = False,
+) -> ValidationResult:
+    """Check for naming standard violations.
+
+    Returns:
+        A ValidationResult object containing messages and violations
+    """
+    logger.info("Validating file: %s", file)
+    if file.suffix not in SUPPORTED_DATASET_FILE_SUFFIXES:
+        logger.info("Skipping validation on non-dataset file: %s", file)
+        return await _ignored_file_type_result(file)
+
+    result = ValidationResult(success=True, file_path=str(file))
+
+    if check_file_exists and not file.exists():
+        result.add_message(
+            FILE_DOES_NOT_EXIST,
+        )
+
+    result.violations = await asyncio.get_running_loop().run_in_executor(
+        None,
+        lambda: _check_violations(file),
+    )
+
+    if result.violations:
+        result.success = False
+        result.add_message(NAME_STANDARD_VIOLATION)
+    else:
+        result.success = True
+        result.add_message(
+            NAME_STANDARD_SUCCESS,
+        )
+    return result
+
+
+async def _ignored_folder_result(file: Path) -> ValidationResult:
+    r = ValidationResult(success=True, file_path=str(file))
+    r.add_message(PATH_IGNORED)
+    return r
+
+
+async def _ignored_file_type_result(file: Path) -> ValidationResult:
+    r = ValidationResult(success=True, file_path=str(file))
+    r.add_message(FILE_IGNORED)
+    return r
+
+
+async def validate_directory(
+    path: Path,
+) -> AsyncGenerator[AsyncGenerator | asyncio.Task]:
+    """Validate a file or recursively validate all files in a directory."""
+    if set(path.parts).intersection(IGNORED_FOLDERS):
+        logger.info("File path ignored: %s", path)
+        yield asyncio.create_task(_ignored_folder_result(path))
+    elif path.suffix:
+        yield asyncio.create_task(_validate_file(path, check_file_exists=True))
+    else:
+        for obj in await asyncio.get_running_loop().run_in_executor(
+            None,
+            lambda: path.glob("*"),
+        ):
+            if obj.suffix:
+                yield asyncio.create_task(_validate_file(obj), name=obj.name)
+            else:
+                logger.debug("Recursing into: %s", obj)
+                yield validate_directory(obj)

dapla_metadata/standards/standard_validators.py

@@ -0,0 +1,98 @@
+import asyncio
+import logging
+import os
+import time
+from collections.abc import AsyncGenerator
+
+from dapla_metadata.datasets.utility.utils import normalize_path
+from dapla_metadata.standards.name_validator import NamingStandardReport
+from dapla_metadata.standards.name_validator import ValidationResult
+from dapla_metadata.standards.name_validator import validate_directory
+
+logger = logging.getLogger(__name__)
+
+
+async def check_naming_standard(
+    file_path: str | os.PathLike[str],
+) -> list[ValidationResult]:
+    """Check whether a given path follows the SSB naming standard.
+
+    This function checks whether the provided `file_path` and subdirectories thereof comply
+    with the naming standard. Currently we only examine '.parquet' files. Other files are ignored.
+
+    Args:
+        file_path: The path to a bucket, directory, or specific file to validate.
+                This can be in the following forms:
+                  - A bucket URL in the form 'gs://ssb-dapla-felles-data-produkt-test'
+                  - An absolute path to a mounted bucket in the form '/buckets/produkt'
+                  - Any subdirectory or file thereof
+                We also accept paths which don't yet exist so that you can test if a path will comply.
+
+    Returns:
+        list[ValidationResult]: A list of validation results,
+        including success status, checked file path, messages, and any detected violations.
+
+    Examples:
+        >>> check_naming_standard("/data/example_file.parquet").success
+        False
+
+        >>> check_naming_standard("/buckets/produkt/datadoc/utdata/person_data_p2021_v2.parquet").success
+        True
+    """
+    results = []
+
+    # Begin validation.
+    # For each file this returns a task which we can wait on to complete.
+    # For each directory this returns another AsyncGenerator which must be unpacked below
+    tasks = [t async for t in validate_directory(normalize_path(str(file_path)))]  # type:ignore [arg-type]
+
+    # 5 minute timeout for safety
+    start_time = time.time()
+    while time.time() < start_time + (5 * 60):
+        for item in tasks:
+            if isinstance(item, AsyncGenerator):
+                # Drill down into lower directories to get the validation tasks from them
+                tasks.remove(item)
+                new_tasks = [t async for t in item]
+                logger.debug("New Tasks: %s %s", len(new_tasks), new_tasks)
+                tasks.extend(
+                    new_tasks,
+                )
+            elif isinstance(item, asyncio.Task):
+                if item.done():
+                    logger.info("Validated %s", item.get_name())
+                    tasks.remove(item)
+                    results.append(item.result())
+
+        logger.debug("Tasks: %s %s", len(tasks), tasks)
+        logger.debug("Results: %s", len(results))
+
+        if len(tasks) == 0:
+            logger.info("Completed validation")
+            break
+
+        # Allow time for other processing to be performed
+        await asyncio.sleep(0.001)
+
+    return results
+
+
+def generate_validation_report(
+    validation_results: list[ValidationResult],
+) -> NamingStandardReport:
+    """Generate and print a formatted naming standard validation report.
+
+    This function takes a list of `ValidationResult` objects, creates a
+    `NamingStandardReport` instance, and prints the generated report.
+
+    Args:
+        validation_results: A list of ValidationResult objects that
+        contain the outcomes of the name standard checks.
+
+    Returns:
+        NamingStandardReport: An instance of `NamingStandardReport` containing
+        the validation results.
+    """
+    report = NamingStandardReport(validation_results=validation_results)
+    print(report.generate_report())  # noqa: T201
+    return report

dapla_metadata/standards/utils/__init__.py

@@ -0,0 +1 @@
+"""Utils for validating ssb standards."""

dapla_metadata/standards/utils/constants.py

@@ -0,0 +1,49 @@
+"""Constants used in validate ssb name standard."""
+
+from dapla_metadata.datasets.dataset_parser import SUPPORTED_DATASET_FILE_SUFFIXES
+
+SUCCESS = "Suksess"
+
+NAME_STANDARD_SUCCESS = "Filene dine er i samsvar med SSB-navnestandarden"
+
+NAME_STANDARD_VIOLATION = "Det er oppdaget brudd på SSB-navnestandard:"
+
+MISSING_BUCKET_NAME = "Filnavn mangler bøttenavn ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
+MISSING_VERSION = "Filnavn mangler versjon ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
+MISSING_PERIOD = "Filnavn mangler gyldighetsperiode ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
+MISSING_SHORT_NAME = "Kortnavn for statistikk mangler ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
+MISSING_DATA_STATE = "Mappe for datatilstand mangler ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#obligatoriske-mapper"
+MISSING_DATASET_SHORT_NAME = "Filnavn mangler datasett kortnavn ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
+
+INVALID_SYMBOLS = "Filnavn inneholder ulovlige tegn ref: https://manual.dapla.ssb.no/statistikkere/navnestandard.html#filnavn"
+
+PATH_IGNORED = "Ignorert, mappen er ikke underlagt krav til navnestandard."
+FILE_IGNORED = f"Ignorert, kun datasett med {', '.join(SUPPORTED_DATASET_FILE_SUFFIXES.keys())} filendelser valideres foreløpig."
+
+FILE_DOES_NOT_EXIST = "Filen eksisterer ikke. Validerer uansett."
+
+BUCKET_NAME_UNKNOWN = "Kan ikke validere bøttenavn"
+
+SSB_NAMING_STANDARD_REPORT = "SSB navnestandard rapport"
+SSB_NAMING_STANDARD_REPORT_SUCCESS_RATE = "Suksess rate"
+SSB_NAMING_STANDARD_REPORT_RESULT_BEST = "🚀 Fantastisk! Alt bestått! 🎉\n"
+SSB_NAMING_STANDARD_REPORT_RESULT_GOOD = (
+    "✅ Bra jobba! Fortsatt litt rom for forbedring. 😊\n"
+)
+SSB_NAMING_STANDARD_REPORT_RESULT_AVERAGE = (
+    "⚠️ Ikke verst! Men det er noen feil å fikse. 🔧\n"
+)
+SSB_NAMING_STANDARD_REPORT_RESULT_LOW = "❌ Mye å forbedre! Ta en grundig sjekk. 🛠️\n"
+SSB_NAMING_STANDARD_REPORT_RESULT_NO_SCORE = "👀 Ingen filer ble validert\n"
+SSB_NAMING_STANDARD_REPORT_FILES = "Antall filer validert"
+SSB_NAMING_STANDARD_REPORT_SUCCESS = "Antall filer som følger SSB navnestandard"
+SSB_NAMING_STANDARD_REPORT_VIOLATIONS = "Antall filer som bryter SSB navnestandard"
+
+IGNORED_FOLDERS = [
+    "temp",
+    "oppdrag",
+    "konfigurasjon",
+    "logg",
+    "tidsserier",
+    "migrert",
+]

dapla_metadata/variable_definitions/_utils/descriptions.py

@@ -2,15 +2,18 @@
 
 import logging
 from pathlib import Path
+from typing import TYPE_CHECKING
 from typing import cast
 
-import yaml
+import ruamel.yaml
 from pydantic import BaseModel
 from pydantic import Field
-from pydantic.config import JsonDict
 
 from dapla_metadata.variable_definitions._utils.config import get_descriptions_path
 
+if TYPE_CHECKING:
+    from pydantic.config import JsonDict
+
 logger = logging.getLogger(__name__)
 
 
@@ -34,7 +37,7 @@ def load_descriptions(file_path: Path) -> dict:
     dict: Parsed contents of the YAML file.
     """
     with Path.open(file_path, encoding="utf-8") as f:
-        return yaml.safe_load(f)
+        return ruamel.yaml.YAML().load(f)
 
 
 def apply_norwegian_descriptions_to_model(
@@ -74,7 +77,7 @@ def apply_norwegian_descriptions_to_model(
             title=field_info.title,
             description=field_info.description,
             json_schema_extra=cast(
-                JsonDict,
+                "JsonDict",
                 {
                     "norwegian_description": new_description,
                     "annotation": field_info.annotation,

dapla_metadata/variable_definitions/variable_definition.py

@@ -1,9 +1,10 @@
 import logging
 from datetime import date
+from io import StringIO
 from os import PathLike
 from pathlib import Path
 
-import yaml
+import ruamel.yaml
 from pydantic import ConfigDict
 from pydantic import PrivateAttr
 
@@ -122,6 +123,8 @@ class VariableDefinition(CompleteResponse):
                 update_draft=update_draft,
             ),
         )
+        self.__dict__.update(updated)
+
         logger.info(
             "Successfully updated variable definition '%s' with ID '%s'",
             updated.short_name,
@@ -228,6 +231,8 @@ class VariableDefinition(CompleteResponse):
                 valid_from=valid_from,
             ),
         )
+        self.__dict__.update(new_patch)
+
         logger.info(
             "Successfully created patch with patch ID '%s' for variable definition '%s' with ID '%s'",
             new_patch.patch_id,
@@ -301,6 +306,7 @@ class VariableDefinition(CompleteResponse):
                 validity_period=validity_period,
             ),
         )
+        self.__dict__.update(new_validity_period)
 
         logger.info(
             "Successfully created validity period that is valid from '%s' for variable definition '%s' with ID '%s'",
@@ -398,13 +404,17 @@ class VariableDefinition(CompleteResponse):
         return self._convert_to_yaml_output()
 
     def _convert_to_yaml_output(self) -> str:
-        return yaml.dump(
-            self.model_dump(
-                mode="json",
-                serialize_as_any=True,
-                warnings="error",
-            ),
-            allow_unicode=True,
-            default_flow_style=False,
-            sort_keys=False,
-        )
+        stream = StringIO()
+        with ruamel.yaml.YAML(
+            output=stream,
+        ) as yaml:
+            yaml.default_flow_style = False
+            yaml.allow_unicode = True
+            yaml.dump(
+                self.model_dump(
+                    mode="json",
+                    serialize_as_any=True,
+                    warnings="error",
+                ),
+            )
+        return stream.getvalue()

{dapla_toolbelt_metadata-0.5.0.dist-info → dapla_toolbelt_metadata-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dapla-toolbelt-metadata
-Version: 0.5.0
+Version: 0.6.0
 Summary: Dapla Toolbelt Metadata
 License: MIT
 Author: Team Metadata
@@ -15,16 +15,16 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: arrow (>=1.3.0)
 Requires-Dist: beautifulsoup4 (>=4.12.3)
 Requires-Dist: cloudpathlib[gs] (>=0.17.0)
-Requires-Dist: dapla-toolbelt (>=3.0.1)
-Requires-Dist: pandas (>=1.4.2)
+Requires-Dist: google-auth (>=2.38.0)
+Requires-Dist: lxml (>=5.3.1)
 Requires-Dist: pyarrow (>=8.0.0)
 Requires-Dist: pydantic (>=2.5.2)
 Requires-Dist: pyjwt (>=2.8.0)
 Requires-Dist: python-dotenv (>=1.0.1)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: ruamel-yaml (>=0.18.10,<0.19.0)
+Requires-Dist: ruamel-yaml (>=0.18.10)
 Requires-Dist: ssb-datadoc-model (>=6.0.0,<7.0.0)
-Requires-Dist: ssb-klass-python (>=0.0.9)
+Requires-Dist: ssb-klass-python (>=1.0.1)
 Requires-Dist: typing-extensions (>=4.12.2)
 Project-URL: Changelog, https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases
 Project-URL: Documentation, https://statisticsnorway.github.io/dapla-toolbelt-metadata

{dapla_toolbelt_metadata-0.5.0.dist-info → dapla_toolbelt_metadata-0.6.0.dist-info}/RECORD

@@ -1,15 +1,15 @@
-dapla_metadata/__init__.py,sha256=5OG8ualI_MkHfkI-I_mrEPaZ2fBJst-VS7i1eNamsxQ,371
+dapla_metadata/__init__.py,sha256=LI-qV1Vq1nKw8KkO0uppNwjOoXwd8niQFCx9jECn6Aw,415
 dapla_metadata/_shared/__init__.py,sha256=qUFgnVhBVlPRQP0ePmY76c8FvWRrJ-9c5GvzibwERnQ,103
 dapla_metadata/_shared/config.py,sha256=QqXcmP66AfXF8wi6FMsa7et7kH2k4EJPOF4IELKuQig,3213
 dapla_metadata/_shared/enums.py,sha256=WHkH1d8xw41gOly6au_izZB1_-6XTcKu5rhBWUImjp8,509
 dapla_metadata/_shared/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dapla_metadata/dapla/__init__.py,sha256=7WVQRBKvpPGyuM0NSjF9sdaups80nYlUdaS1wCZjp4g,57
-dapla_metadata/dapla/user_info.py,sha256=pY86hnlaSzRAtnYchF27Na2nSK8-A6ZOiub7P2zr2tU,4775
+dapla_metadata/dapla/__init__.py,sha256=tkapF-YwmruPPrKvN3pEoCZqb7xvJx_ogBM8XyGMuJI,130
+dapla_metadata/dapla/user_info.py,sha256=bENez-ICt9ySR8orYebO68Q3_2LkIW9QTL58DTctmEQ,4833
 dapla_metadata/datasets/__init__.py,sha256=TvzskpdFC6hGcC9_55URT5jr5wNAPzXuISd2UjJWM_8,280
 dapla_metadata/datasets/code_list.py,sha256=gNSJsDoQsOgwr-nivNoV51cf19ar_I_Qd9S8LXhdJVE,9065
 dapla_metadata/datasets/core.py,sha256=gp-zTJyn5Z_9bbmHF749OUx0kpYviiDAP9MDwSZiRW8,21768
-dapla_metadata/datasets/dapla_dataset_path_info.py,sha256=drPB7EXhAGMSIktX9DfjGoCstX_RQhgiqdk1N-9Lkhg,22058
-dapla_metadata/datasets/dataset_parser.py,sha256=12zkxGuBFXDNKoLnGRM6RfDWLY6ryWM366t4c_tNSbg,8007
+dapla_metadata/datasets/dapla_dataset_path_info.py,sha256=zdkVjxlqXMBe7eTAneUrTDP0_fx7JsEQ_0JrKjREhfU,26854
+dapla_metadata/datasets/dataset_parser.py,sha256=8lFsmxvFOgNG8YLv908lmsnSAYigv8PBMdtBYD2gteI,8145
 dapla_metadata/datasets/external_sources/__init__.py,sha256=qvIdXwqyEmXNUCB94ZtZXRzifdW4hiXASFFPtC70f6E,83
 dapla_metadata/datasets/external_sources/external_sources.py,sha256=9eIcOIUbaodNX1w9Tj2wl4U4wUmr5kF1R0i01fKUzGs,2974
 dapla_metadata/datasets/model_backwards_compatibility.py,sha256=0NZM0j3P3ngmssH69t24BKagAxEL3veb_n5TkWK2jaA,19167
@@ -19,7 +19,12 @@ dapla_metadata/datasets/statistic_subject_mapping.py,sha256=QdC22DUBOdRgsfmTTEUr
 dapla_metadata/datasets/utility/__init__.py,sha256=pp6tUcgUbo8iq9OPtFKQrTbLuI3uY7NHptwWSTpasOU,33
 dapla_metadata/datasets/utility/constants.py,sha256=SqZMc1v8rO2b_nRFJR7frVd0TAGvvxzIPEIzkqOuBSw,2444
 dapla_metadata/datasets/utility/enums.py,sha256=SpV4xlmP1YMaJPbmX03hqRLHUOhXIk5gquTeJ8G_5OE,432
-dapla_metadata/datasets/utility/utils.py,sha256=C3xCYRiaPutYxrkVakkpNDJheSqTJNVqxdC7qKHLXao,18384
+dapla_metadata/datasets/utility/utils.py,sha256=belHeJ3a8PWtfWRQoGSsoMPu72VAhryLnlYR7gg9kZM,18361
+dapla_metadata/standards/__init__.py,sha256=n8jnMrudLuScSdfQ4UMJorc-Ptg3Y1-ilT8zAaQnM70,179
+dapla_metadata/standards/name_validator.py,sha256=VercKsO7bNDEfAXy_oxMadL8KLVhxmmTe0T8sNlwzm0,9179
+dapla_metadata/standards/standard_validators.py,sha256=tcCiCI76wUVtMzXA2oCgdauZc0uGgUi11FKu-t7KGwQ,3767
+dapla_metadata/standards/utils/__init__.py,sha256=AiM7JcpFsAgyuCyLDYZo9kI94wvIImMDGoV2lKhS4pE,42
+dapla_metadata/standards/utils/constants.py,sha256=SiWViQo90PXruWeACuW9Y1iNqmFoDVe0dP4WfO29PXU,2498
 dapla_metadata/variable_definitions/__init__.py,sha256=j_Nn5mnlZ2uio9moDFLE2xpALqrYpupIZMlvwbLuEuA,391
 dapla_metadata/variable_definitions/_generated/.openapi-generator/FILES,sha256=hfNllHEkFODP0XbgqZB5Tz2mmEBFeAeMplXXslczo1E,634
 dapla_metadata/variable_definitions/_generated/.openapi-generator/VERSION,sha256=Y6lrqS2bXoujk5K-DCAwRFdRmkCKuTgvlngEx6FY5So,7
@@ -70,15 +75,15 @@ dapla_metadata/variable_definitions/_utils/__init__.py,sha256=qAhRLJoTBqtR3f9xRX
 dapla_metadata/variable_definitions/_utils/_client.py,sha256=v1-9VjrdPI6-sroam5vXMPEV1dQMPsYk7KyGd48HjYw,971
 dapla_metadata/variable_definitions/_utils/config.py,sha256=PZIe0XsrYKsF5SiATCx5EoSn-s9EbYAteiA2k5FxRTw,2475
 dapla_metadata/variable_definitions/_utils/constants.py,sha256=b0HOSPeTbY0Zjs10YIOMDx8MsSQF12ES9tvXlj8VTao,1483
-dapla_metadata/variable_definitions/_utils/descriptions.py,sha256=41Rrcn38Pq9h0Up48rKPdLCmZF1oyqf2DSVqodbLGSA,2672
+dapla_metadata/variable_definitions/_utils/descriptions.py,sha256=bB5QHNc4eOhmpLQHCty-CP5_aA82chkICifXw430suI,2746
 dapla_metadata/variable_definitions/_utils/files.py,sha256=sK6rYKJLGp3nSySLPM9WX5SFOFSit_envi9knUmZWi0,9253
 dapla_metadata/variable_definitions/_utils/template_files.py,sha256=hCINDyGFYJtFaWP-0Qjbrv0S6GFkPccIDyKD0KcPtuA,3670
 dapla_metadata/variable_definitions/_utils/variable_definition_files.py,sha256=ePlbsrVl1JNMDUomS-ldYOeOilmcjQy0I5RhorShE2o,2785
 dapla_metadata/variable_definitions/exceptions.py,sha256=z6Gtd84FboDu7vWjC3wathIF7I0gF0imtRhwMkr16lY,7851
 dapla_metadata/variable_definitions/resources/vardef_model_descriptions_nb.yaml,sha256=cA1jh6JBTPWluSc2fhC__lMj8wmDrlk3CJODzSY8nKY,4771
 dapla_metadata/variable_definitions/vardef.py,sha256=ZBV8ezGRJ8rsmb2H-ZyVX3-DNnMDPIZsHa7MzlcEkNM,11340
-dapla_metadata/variable_definitions/variable_definition.py,sha256=umplTS6bgD2iAebg0WLaO2j04IVGb3R0RLpLqwmj0bI,14481
-dapla_toolbelt_metadata-0.5.0.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
-dapla_toolbelt_metadata-0.5.0.dist-info/METADATA,sha256=TqCKEYq9pNbU2DVbAXTe2malm6mEOMzLHpaRrq2TLjI,4878
-dapla_toolbelt_metadata-0.5.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-dapla_toolbelt_metadata-0.5.0.dist-info/RECORD,,
+dapla_metadata/variable_definitions/variable_definition.py,sha256=Q_XjMKaJhPwoM0vyfgYVWKYNAYFX0caNWWthXmQxeNs,14784
+dapla_toolbelt_metadata-0.6.0.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
+dapla_toolbelt_metadata-0.6.0.dist-info/METADATA,sha256=_3-JxJStSrW_2AF960_zzYuvC3iVKleQkrFbs-520cQ,4866
+dapla_toolbelt_metadata-0.6.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+dapla_toolbelt_metadata-0.6.0.dist-info/RECORD,,