dapla-toolbelt-metadata 0.5.0__tar.gz → 0.6.1__tar.gz

{dapla_toolbelt_metadata-0.5.0 → dapla_toolbelt_metadata-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dapla-toolbelt-metadata
-Version: 0.5.0
+Version: 0.6.1
 Summary: Dapla Toolbelt Metadata
 License: MIT
 Author: Team Metadata
@@ -15,16 +15,16 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: arrow (>=1.3.0)
 Requires-Dist: beautifulsoup4 (>=4.12.3)
 Requires-Dist: cloudpathlib[gs] (>=0.17.0)
-Requires-Dist: dapla-toolbelt (>=3.0.1)
-Requires-Dist: pandas (>=1.4.2)
+Requires-Dist: google-auth (>=2.38.0)
+Requires-Dist: lxml (>=5.3.1)
 Requires-Dist: pyarrow (>=8.0.0)
 Requires-Dist: pydantic (>=2.5.2)
 Requires-Dist: pyjwt (>=2.8.0)
 Requires-Dist: python-dotenv (>=1.0.1)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: ruamel-yaml (>=0.18.10,<0.19.0)
+Requires-Dist: ruamel-yaml (>=0.18.10)
 Requires-Dist: ssb-datadoc-model (>=6.0.0,<7.0.0)
-Requires-Dist: ssb-klass-python (>=0.0.9)
+Requires-Dist: ssb-klass-python (>=1.0.1)
 Requires-Dist: typing-extensions (>=4.12.2)
 Project-URL: Changelog, https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases
 Project-URL: Documentation, https://statisticsnorway.github.io/dapla-toolbelt-metadata

{dapla_toolbelt_metadata-0.5.0 → dapla_toolbelt_metadata-0.6.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dapla-toolbelt-metadata"
-version = "0.5.0"
+version = "0.6.1"
 description = "Dapla Toolbelt Metadata"
 authors = ["Team Metadata <metadata@ssb.no>"]
 license = "MIT"
@@ -19,18 +19,18 @@ Changelog = "https://github.com/statisticsnorway/dapla-toolbelt-metadata/release
 python = ">=3.11,<4.0"
 pyarrow = ">=8.0.0"
 pydantic = ">=2.5.2"
-pandas = ">=1.4.2"
-dapla-toolbelt = ">=3.0.1"
 arrow = ">=1.3.0"
 python-dotenv = ">=1.0.1"
 requests = ">=2.31.0"
 beautifulsoup4 = ">=4.12.3"
 cloudpathlib = { extras = ["gs"], version = ">=0.17.0" }
 pyjwt = ">=2.8.0"
-ssb-klass-python = ">=0.0.9"
+ssb-klass-python = ">=1.0.1"
 ssb-datadoc-model = "^6.0.0"
 typing-extensions = ">=4.12.2"
-ruamel-yaml = "^0.18.10"
+ruamel-yaml = ">=0.18.10"
+google-auth = ">=2.38.0"
+lxml = ">=5.3.1"
 
 [tool.poetry.group.dev.dependencies]
 pygments = ">=2.18.0"
@@ -48,6 +48,7 @@ typeguard = ">=2.13.3"
 xdoctest = { extras = ["colors"], version = ">=0.15.10" }
 myst-parser = { version = ">=0.16.1" }
 mypy = ">=0.950"
+pandas = ">=1.4.2"
 pytest-cov = ">=3.0.0"
 python-kacl = "*"
 pytest-mock = ">=3.14.0"
@@ -58,11 +59,13 @@ requests-mock = ">=1.12.1"
 types-Pygments = "*"
 types-colorama = "*"
 types-setuptools = "*"
+types-requests = "*"
 types-beautifulsoup4 = ">=4.12.0.20240511"
 ipykernel = "^6.29.5"
 rich = "^13.9.4"
 bpython = "^0.24"
 testcontainers = { version = "^4.8.2", extras = ["generic"] }
+pytest-asyncio = "^0.26.0"
 
 [tool.pytest.ini_options]
 pythonpath = ["src"]
@@ -115,8 +118,8 @@ module = [
     "datadoc_model.model",
     "pytest_mock",
     "testcontainers.*",
-    "yaml",
     "httpx",
+    "ruamel.*",
 ]
 ignore_missing_imports = true
 
@@ -152,9 +155,11 @@ select = ["ALL"]
 ignore = [
     "ANN202",  # Don't requiere return type annotation for private functions.
     "ANN401",  # Allow type annotation with type Any.
+    "COM812",  # Suggested to ignore when using ruff format
     "D100",    # Supress undocumented-public-module. Only doc of public api required.
     "FBT001",  # Allow boolean positional arguments in a function.
     "FBT002",  # Allow boolean default positional arguments in a function.
+    "FIX002",  # Don't fail on TODO as long as it's documented correctly.
     "E402",    # Supress module-import-not-at-top-of-file, needed in jupyter notebooks.
     "E501",    # Supress line-too-long warnings: trust black's judgement on this one.
     "PLR2004", # Allow to compare with unnamed numerical constants.

{dapla_toolbelt_metadata-0.5.0 → dapla_toolbelt_metadata-0.6.1}/src/dapla_metadata/__init__.py

@@ -9,5 +9,7 @@ warnings.filterwarnings(
 
 import datadoc_model.model as datadoc_model
 
+from . import dapla
 from . import datasets
+from . import standards
 from . import variable_definitions

dapla_toolbelt_metadata-0.6.1/src/dapla_metadata/dapla/__init__.py

@@ -0,0 +1,4 @@
+"""Expose information specific to the Dapla platform."""
+
+from .user_info import DaplaLabUserInfo
+from .user_info import UserInfo

{dapla_toolbelt_metadata-0.5.0 → dapla_toolbelt_metadata-0.6.1}/src/dapla_metadata/dapla/user_info.py

@@ -13,11 +13,6 @@ from dapla_metadata._shared.enums import DaplaService
 logger = logging.getLogger(__name__)
 
 
-PLACEHOLDER_EMAIL_ADDRESS = "default_user@ssb.no"
-PLACEHOLDER_GROUP = "default-team-developers"
-PLACEHOLDER_TEAM = "default-team"
-
-
 class UserInfo(Protocol):
     """Information about the current user.
 
@@ -62,20 +57,24 @@ class UnknownUserInfo:
 class TestUserInfo:
     """Information about the current user for local development and testing."""
 
+    PLACEHOLDER_EMAIL_ADDRESS = "default_user@ssb.no"
+    PLACEHOLDER_GROUP = "default-team-developers"
+    PLACEHOLDER_TEAM = "default-team"
+
     @property
     def short_email(self) -> str | None:
         """Get the short email address."""
-        return PLACEHOLDER_EMAIL_ADDRESS
+        return TestUserInfo.PLACEHOLDER_EMAIL_ADDRESS
 
     @property
     def current_group(self) -> str | None:
         """Get the group which the user is currently representing."""
-        return PLACEHOLDER_GROUP
+        return TestUserInfo.PLACEHOLDER_GROUP
 
     @property
     def current_team(self) -> str | None:
         """Get the team which the user is currently representing."""
-        return PLACEHOLDER_TEAM
+        return TestUserInfo.PLACEHOLDER_TEAM
 
 
 class DaplaLabUserInfo:
@@ -146,17 +145,17 @@ def get_user_info_for_current_platform() -> UserInfo:
 def parse_team_name(group: str) -> str:
     """Parses the group to get the current team.
 
-    >>> parse_team_name(dapla-metadata-developers)
-    (dapla-metadata)
+    >>> parse_team_name("dapla-metadata-developers")
+    'dapla-metadata'
 
-    >>> parse_team_name(dapla-metadata-data-admins)
-    (dapla-metadata)
+    >>> parse_team_name("dapla-metadata-data-admins")
+    'dapla-metadata'
 
-    >>> parse_team_name(dapla-metadata)
-    (dapla)
+    >>> parse_team_name("dapla-metadata")
+    'dapla'
 
-    >>> parse_team_name(dapla-metadata-not-real-name)
-    (dapla-metadata-not-real)
+    >>> parse_team_name("dapla-metadata-not-real-name")
+    'dapla-metadata-not-real'
     """
     parts = group.split("-")
     return "-".join(parts[:-2] if group.endswith("data-admins") else parts[:-1])

{dapla_toolbelt_metadata-0.5.0 → dapla_toolbelt_metadata-0.6.1}/src/dapla_metadata/datasets/dapla_dataset_path_info.py

@@ -478,7 +478,7 @@ class DaplaDatasetPathInfo:
         """Extract the bucket name from the dataset path.
 
         Returns:
-            The bucket name or None if the dataset path is not a GCS path.
+            The bucket name or None if the dataset path is not a GCS path nor ssb bucketeer path.
 
         Examples:
             >>> DaplaDatasetPathInfo('gs://ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
@@ -492,17 +492,35 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
             None
+
+            >>> DaplaDatasetPathInfo('ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').bucket_name
+            None
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-data-delt
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-data-delt
+
+            >>> DaplaDatasetPathInfo('home/work/buckets/ssb-staging-dapla-felles-produkt/stat/utdata/person_data_p2021_v2.parquet').bucket_name
+            ssb-staging-dapla-felles-produkt
         """
         prefix: str | None = None
-        if self.dataset_string.startswith(GSPath.cloud_prefix):
+        dataset_string = str(self.dataset_string)
+        if GSPath.cloud_prefix in self.dataset_string:
             prefix = GSPath.cloud_prefix
-        elif self.dataset_string.startswith(GS_PREFIX_FROM_PATHLIB):
+            _, bucket_and_rest = dataset_string.split(prefix, 1)
+        elif GS_PREFIX_FROM_PATHLIB in self.dataset_string:
             prefix = GS_PREFIX_FROM_PATHLIB
+            _, bucket_and_rest = self.dataset_string.split(prefix, 1)
+        elif "buckets/" in self.dataset_string:
+            prefix = "buckets/"
+            _, bucket_and_rest = self.dataset_string.split(prefix, 1)
         else:
             return None
 
         return pathlib.Path(
-            self.dataset_string.removeprefix(prefix),
+            bucket_and_rest,
         ).parts[0]
 
     @property
@@ -528,6 +546,15 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('my_data/simple_dataset_name.parquet').dataset_short_name
             simple_dataset_name
+
+            >>> DaplaDatasetPathInfo('gs:/ssb-staging-dapla-felles-data-delt/datadoc/utdata/person_data_p2021_v2.parquet').dataset_short_name
+            person_data
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/folk_data_p2021_v2.parquet').dataset_short_name
+            folk_data
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/stat/utdata/dapla/bus_p2021_v2.parquet').dataset_short_name
+            bus
         """
         if self.contains_data_from or self.contains_data_until:
             short_name_sections = self.dataset_name_sections[
@@ -601,6 +628,9 @@ class DaplaDatasetPathInfo:
             >>> DaplaDatasetPathInfo('utdata/min_statistikk/person_data_v1.parquet').dataset_state
             <DataSetState.OUTPUT_DATA: 'OUTPUT_DATA'>
 
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data_v1.parquet').dataset_state
+            <DataSetState.INPUT_DATA: 'INPUT_DATA'>
+
             >>> DaplaDatasetPathInfo('my_special_data/person_data_v1.parquet').dataset_state
             None
         """
@@ -632,6 +662,12 @@ class DaplaDatasetPathInfo:
 
             >>> DaplaDatasetPathInfo('person_data.parquet').dataset_version
             None
+
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data_v1.parquet').dataset_version
+            '1'
+
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data.parquet').dataset_version
+            None
         """
         minimum_elements_in_file_name: Final[int] = 2
         minimum_characters_in_version_string: Final[int] = 2
@@ -645,13 +681,37 @@ class DaplaDatasetPathInfo:
                 return last_filename_element[1:]
         return None
 
+    def _get_left_parts(
+        self,
+        dataset_path_parts: list[str],
+        state_index: int,
+    ) -> list[str]:
+        """Retrieve the path parts before the dataset state, considering bucket prefixes."""
+        bucket_prefix = {"gs:", "buckets"}
+        left_parts = dataset_path_parts[:state_index]
+
+        # Stop checking beyond the bucket prefix
+        prefix_intersection = bucket_prefix & set(left_parts)
+        if prefix_intersection:
+            first_prefix = min(
+                left_parts.index(prefix) for prefix in prefix_intersection
+            )
+            left_parts = left_parts[first_prefix:]
+
+        return (
+            []
+            if left_parts == ["/"]
+            or (left_parts[0] in bucket_prefix and len(left_parts) <= 2)
+            else left_parts
+        )
+
     @property
     def statistic_short_name(
         self,
     ) -> str | None:
         """Extract the statistical short name from the filepath.
 
-        Extract the statistical short name from the filepath right before the
+        Extract the statistical short name from the filepath either after bucket name or right before the
         dataset state based on the Dapla filepath naming convention.
 
         Returns:
@@ -662,21 +722,75 @@ class DaplaDatasetPathInfo:
             >>> DaplaDatasetPathInfo('prosjekt/befolkning/klargjorte_data/person_data_v1.parquet').statistic_short_name
             befolkning
 
+            >>> DaplaDatasetPathInfo('buckets/prosjekt/befolkning/person_data_v1.parquet').statistic_short_name
+            befolkning
+
             >>> DaplaDatasetPathInfo('befolkning/inndata/person_data_v1.parquet').statistic_short_name
             befolkning
 
+            >>> DaplaDatasetPathInfo('buckets/bucket_name/stat_name/inndata/min_statistikk/person_data.parquet').statistic_short_name
+            stat_name
+
+            >>> DaplaDatasetPathInfo('buckets/stat_name/utdata/person_data.parquet').statistic_short_name
+            None
+
             >>> DaplaDatasetPathInfo('befolkning/person_data.parquet').statistic_short_name
             None
+
+            >>> DaplaDatasetPathInfo('buckets/produkt/befolkning/utdata/person_data.parquet').statistic_short_name
+            befolkning
+
+            >>> DaplaDatasetPathInfo('resources/buckets/produkt/befolkning/utdata/person_data.parquet').statistic_short_name
+            befolkning
+
+            >>> DaplaDatasetPathInfo('gs://statistikk/produkt/klargjorte-data/persondata_p1990-Q1_p2023-Q4_v1/aar=2019/data.parquet').statistic_short_name
+            produkt
+
+            >>> DaplaDatasetPathInfo('gs://statistikk/produkt/persondata_p1990-Q1_p2023-Q4_v1/aar=2019/data.parquet').statistic_short_name
+            None
+
+            >>> DaplaDatasetPathInfo('buckets/ssb-staging-dapla-felles-data-delt/person_data_p2021_v2.parquet').statistic_short_name
+            None
         """
-        dataset_state = self.dataset_state
-        if dataset_state is not None:
-            dataset_state_names = self._extract_norwegian_dataset_state_path_part(
-                dataset_state,
-            )
-            dataset_path_parts = list(self.dataset_path.parts)
-            for i in dataset_state_names:
-                if i in dataset_path_parts and dataset_path_parts.index(i) != 0:
-                    return dataset_path_parts[dataset_path_parts.index(i) - 1]
+        if not self.dataset_state:
+            if self.bucket_name:
+                parts = self.dataset_path.parent.parts
+
+                if self.bucket_name not in parts:
+                    return None
+
+                # Find the index of bucket_name in the path
+                bucket_name_index = self.dataset_path.parent.parts.index(
+                    self.bucket_name,
+                )
+
+                # If there are parts after bucket_name, return the part immediately after it
+                if len(self.dataset_path.parent.parts) > bucket_name_index + 1:
+                    return self.dataset_path.parent.parts[bucket_name_index + 1]
+
+            return None
+
+        dataset_state_names = self._extract_norwegian_dataset_state_path_part(
+            self.dataset_state,
+        )
+        dataset_path_parts = list(self.dataset_path.parts)
+
+        for state in dataset_state_names:
+            if state not in dataset_path_parts:
+                continue
+
+            index = dataset_path_parts.index(state)
+
+            if index == 0:
+                continue
+
+            left_parts = self._get_left_parts(dataset_path_parts, index)
+
+            if not left_parts:
+                return None
+
+            return dataset_path_parts[index - 1]
+
         return None
 
     def path_complies_with_naming_standard(self) -> bool:

{dapla_toolbelt_metadata-0.5.0 → dapla_toolbelt_metadata-0.6.1}/src/dapla_metadata/datasets/dataset_parser.py

@@ -7,7 +7,6 @@ from __future__ import annotations
 
 import pathlib  # noqa: TC003 import is needed for docs build
 import re
-import typing as t
 from abc import ABC
 from abc import abstractmethod
 from typing import TYPE_CHECKING
@@ -56,6 +55,8 @@ KNOWN_FLOAT_TYPES = (
 
 KNOWN_STRING_TYPES = (
     "string",
+    "string[pyarrow]",
+    "large_string",
     "str",
     "char",
     "varchar",
@@ -67,13 +68,18 @@ KNOWN_STRING_TYPES = (
 
 KNOWN_DATETIME_TYPES = (
     "timestamp",
+    "timestamp[s]",
+    "timestamp[ms]",
     "timestamp[us]",
     "timestamp[ns]",
+    "datetime",
     "datetime64",
-    " datetime64[ns]",
-    " datetime64[us]",
+    "datetime64[s]",
+    "datetime64[ms]",
+    "datetime64[us]",
+    "datetime64[ns]",
     "date",
-    "datetime",
+    "date32[day]",
     "time",
 )
 
@@ -89,9 +95,7 @@ TYPE_CORRESPONDENCE: list[tuple[tuple[str, ...], DataType]] = [
 ]
 TYPE_MAP: dict[str, DataType] = {}
 for concrete_type, abstract_type in TYPE_CORRESPONDENCE:
-    TYPE_MAP.update({c: abstract_type for c in concrete_type})
-
-TDatasetParser = t.TypeVar("TDatasetParser", bound="DatasetParser")
+    TYPE_MAP.update(dict.fromkeys(concrete_type, abstract_type))
 
 
 class DatasetParser(ABC):
@@ -112,31 +116,23 @@ class DatasetParser(ABC):
     @staticmethod
     def for_file(dataset: pathlib.Path | CloudPath) -> DatasetParser:
         """Return the correct subclass based on the given dataset file."""
-        supported_file_types: dict[
-            str,
-            type[DatasetParser],
-        ] = {
-            ".parquet": DatasetParserParquet,
-            ".sas7bdat": DatasetParserSas7Bdat,
-            ".parquet.gzip": DatasetParserParquet,
-        }
         file_type = "Unknown"
         try:
             file_type = dataset.suffix
             # Gzipped parquet files can be read with DatasetParserParquet
-            match = re.search(r"(.parquet.gzip)", str(dataset).lower())
-            file_type = ".parquet.gzip" if match else file_type
-            # Extract the appropriate reader class from the SUPPORTED_FILE_TYPES dict and return an instance of it
-            reader = supported_file_types[file_type](dataset)
+            match = re.search(PARQUET_GZIP_FILE_SUFFIX, str(dataset).lower())
+            file_type = PARQUET_GZIP_FILE_SUFFIX if match else file_type
+            # Extract the appropriate reader class from the SUPPORTED_FILE_TYPES dict
+            reader = SUPPORTED_DATASET_FILE_SUFFIXES[file_type](dataset)
         except IndexError as e:
             # Thrown when just one element is returned from split, meaning there is no file extension supplied
-            msg = f"Could not recognise file type for provided {dataset = }. Supported file types are: {', '.join(supported_file_types.keys())}"
+            msg = f"Could not recognise file type for provided {dataset = }. Supported file types are: {', '.join(SUPPORTED_DATASET_FILE_SUFFIXES.keys())}"
             raise FileNotFoundError(
                 msg,
             ) from e
         except KeyError as e:
             # In this case the file type is not supported, so we throw a helpful exception
-            msg = f"{file_type = } is not supported. Please open one of the following supported files types: {', '.join(supported_file_types.keys())} or contact the maintainers to request support."
+            msg = f"{file_type = } is not supported. Please open one of the following supported files types: {', '.join(SUPPORTED_DATASET_FILE_SUFFIXES.keys())} or contact the maintainers to request support."
             raise NotImplementedError(
                 msg,
             ) from e
@@ -157,6 +153,9 @@ class DatasetParser(ABC):
 
         Arguments:
             data_type: The concrete data type to map.
+
+        Returns:
+            The abstract data type or None
         """
         return TYPE_MAP.get(data_type.lower(), None)
 
@@ -239,3 +238,17 @@ class DatasetParserSas7Bdat(DatasetParser):
             )
 
         return fields
+
+
+PARQUET_FILE_SUFFIX = ".parquet"
+PARQUET_GZIP_FILE_SUFFIX = ".parquet.gzip"
+SAS7BDAT_FILE_SUFFIX = ".sas7bdat"
+
+SUPPORTED_DATASET_FILE_SUFFIXES: dict[
+    str,
+    type[DatasetParser],
+] = {
+    PARQUET_FILE_SUFFIX: DatasetParserParquet,
+    PARQUET_GZIP_FILE_SUFFIX: DatasetParserParquet,
+    SAS7BDAT_FILE_SUFFIX: DatasetParserSas7Bdat,
+}

{dapla_toolbelt_metadata-0.5.0 → dapla_toolbelt_metadata-0.6.1}/src/dapla_metadata/datasets/utility/utils.py

@@ -5,6 +5,7 @@ import logging
 import pathlib
 import uuid
 
+import google.auth
 from cloudpathlib import CloudPath
 from cloudpathlib import GSClient
 from cloudpathlib import GSPath
@@ -13,7 +14,6 @@ from datadoc_model.model import Assessment
 from datadoc_model.model import DataSetState
 from datadoc_model.model import VariableRole
 
-from dapla import AuthClient
 from dapla_metadata.dapla import user_info
 from dapla_metadata.datasets.utility.constants import (
     DATASET_FIELDS_FROM_EXISTING_METADATA,
@@ -52,7 +52,7 @@ def normalize_path(path: str) -> pathlib.Path | CloudPath:
         Pathlib compatible object.
     """
     if path.startswith(GSPath.cloud_prefix):
-        client = GSClient(credentials=AuthClient.fetch_google_credentials())
+        client = GSClient(credentials=google.auth.default()[0])
         return GSPath(path, client=client)
     return pathlib.Path(path)
 

dapla_toolbelt_metadata-0.6.1/src/dapla_metadata/standards/__init__.py

@@ -0,0 +1,4 @@
+"""Expose information specific to validating ssb standards."""
+
+from .standard_validators import check_naming_standard
+from .standard_validators import generate_validation_report