dapla-toolbelt-metadata 0.1.1__py3-none-any.whl

dapla_toolbelt_metadata-0.1.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright © 2024 Statistics Norway
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dapla_toolbelt_metadata-0.1.1.dist-info/METADATA

@@ -0,0 +1,125 @@
+Metadata-Version: 2.1
+Name: dapla-toolbelt-metadata
+Version: 0.1.1
+Summary: Dapla Toolbelt Metadata
+Home-page: https://github.com/statisticsnorway/dapla-toolbelt-metadata
+License: MIT
+Author: Team Metadata
+Author-email: metadata@ssb.no
+Requires-Python: >=3.10,<4.0
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: arrow (>=1.3.0)
+Requires-Dist: beautifulsoup4 (>=4.12.3)
+Requires-Dist: black (>=24.8.0,<25.0.0)
+Requires-Dist: bs4 (>=0.0.2,<0.0.3)
+Requires-Dist: click (>=8.0.1)
+Requires-Dist: cloudpathlib[gs] (>=0.17.0)
+Requires-Dist: coverage (>=7.6.1,<8.0.0)
+Requires-Dist: dapla-toolbelt (>=1.3.3)
+Requires-Dist: faker (>=26.1.0,<27.0.0)
+Requires-Dist: furo (>=2024.7.18,<2025.0.0)
+Requires-Dist: gunicorn (>=21.2.0)
+Requires-Dist: pandas (>=1.4.2)
+Requires-Dist: pre-commit (>=3.8.0,<4.0.0)
+Requires-Dist: pyarrow (>=8.0.0)
+Requires-Dist: pydantic (>=2.5.2)
+Requires-Dist: pygments (>=2.18.0,<3.0.0)
+Requires-Dist: pyjwt (>=2.8.0)
+Requires-Dist: pytest (>=8.3.2,<9.0.0)
+Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
+Requires-Dist: python-dotenv (>=1.0.1)
+Requires-Dist: requests (>=2.31.0)
+Requires-Dist: requests-mock (>=1.12.1,<2.0.0)
+Requires-Dist: ruff (>=0.5.6,<0.6.0)
+Requires-Dist: ssb-datadoc-model (>=6.0.0,<7.0.0)
+Requires-Dist: ssb-klass-python (>=0.0.9)
+Requires-Dist: types-beautifulsoup4 (>=4.12.0.20240511,<5.0.0.0)
+Project-URL: Changelog, https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases
+Project-URL: Documentation, https://statisticsnorway.github.io/dapla-toolbelt-metadata
+Project-URL: Repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata
+Description-Content-Type: text/markdown
+
+# Dapla Toolbelt Metadata
+
+[![PyPI](https://img.shields.io/pypi/v/dapla-toolbelt-metadata.svg)][pypi status]
+[![Status](https://img.shields.io/pypi/status/dapla-toolbelt-metadata.svg)][pypi status]
+[![Python Version](https://img.shields.io/pypi/pyversions/dapla-toolbelt-metadata)][pypi status]
+[![License](https://img.shields.io/pypi/l/dapla-toolbelt-metadata)][license]
+
+[![Documentation](https://github.com/statisticsnorway/dapla-toolbelt-metadata/actions/workflows/docs.yml/badge.svg)][documentation]
+[![Tests](https://github.com/statisticsnorway/dapla-toolbelt-metadata/actions/workflows/tests.yml/badge.svg)][tests]
+[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=statisticsnorway_dapla-toolbelt-metadata&metric=coverage&token=ee0a4d273db2bf811222fca5dffccd4592e3d536)][sonarcov]
+[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=statisticsnorway_dapla-toolbelt-metadata&metric=alert_status&token=ee0a4d273db2bf811222fca5dffccd4592e3d536)][sonarquality]
+
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)][pre-commit]
+[![Black](https://img.shields.io/badge/code%20style-black-000000.svg)][black]
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)][poetry]
+
+[pypi status]: https://pypi.org/project/dapla-toolbelt-metadata/
+[documentation]: https://statisticsnorway.github.io/dapla-toolbelt-metadata
+[tests]: https://github.com/statisticsnorway/dapla-toolbelt-metadata/actions?workflow=Tests
+
+[sonarcov]: https://sonarcloud.io/summary/overall?id=statisticsnorway_dapla-toolbelt-metadata
+[sonarquality]: https://sonarcloud.io/summary/overall?id=statisticsnorway_dapla-toolbelt-metadata
+[pre-commit]: https://github.com/pre-commit/pre-commit
+[black]: https://github.com/psf/black
+[poetry]: https://python-poetry.org/
+
+## Features
+
+- TODO
+
+## Requirements
+
+- TODO
+- Python
+
+## Installation
+
+You can install _Dapla Toolbelt Metadata_ via [pip] from [PyPI]:
+
+```console
+pip install dapla-toolbelt-metadata
+```
+
+## Usage
+
+Please see the [Reference Guide] for details.
+
+## Contributing
+
+Contributions are very welcome.
+To learn more, see the [Contributor Guide].
+
+## License
+
+Distributed under the terms of the [MIT license][license],
+_Dapla Toolbelt Metadata_ is free and open source software.
+
+## Issues
+
+If you encounter any problems,
+please [file an issue] along with a detailed description.
+
+## Credits
+
+This project was generated from [Statistics Norway]'s [SSB PyPI Template].
+
+[statistics norway]: https://www.ssb.no/en
+[pypi]: https://pypi.org/
+[ssb pypi template]: https://github.com/statisticsnorway/ssb-pypitemplate
+[file an issue]: https://github.com/statisticsnorway/dapla-toolbelt-metadata/issues
+[pip]: https://pip.pypa.io/
+
+<!-- github-only -->
+
+[license]: https://github.com/statisticsnorway/dapla-toolbelt-metadata/blob/main/LICENSE
+[contributor guide]: https://github.com/statisticsnorway/dapla-toolbelt-metadata/blob/main/CONTRIBUTING.md
+[reference guide]: https://statisticsnorway.github.io/dapla-toolbelt-metadata/reference.html
+

dapla_toolbelt_metadata-0.1.1.dist-info/RECORD

@@ -0,0 +1,21 @@
+dataset/__init__.py,sha256=aa6dF2ddfeljw1SElaVqAF6YewuZ0Fxk3q7iebtWR2E,378
+dataset/code_list.py,sha256=SqzassTXL-Gr4TqomXStJFiI5gSN0lXjbmZLDJadMrQ,9050
+dataset/config.py,sha256=EWTmrkLWYKSHVrzEWQO16_CaWrxiUllAnRJFbvEoono,4239
+dataset/core.py,sha256=E7OmFM1iTKlKuvQXuwhucO3z5pKg8fTkE2TmOYFi7_M,22654
+dataset/dapla_dataset_path_info.py,sha256=7wwVwykJUaRbqCZrAMsZsOd1p_xO8bHe5LhNOLE8j6k,21600
+dataset/dataset_parser.py,sha256=AvN4cKaDvP4VwplNR5uvXJdiZh4ippNcFTBll-HhH-4,7949
+dataset/external_sources/__init__.py,sha256=qvIdXwqyEmXNUCB94ZtZXRzifdW4hiXASFFPtC70f6E,83
+dataset/external_sources/external_sources.py,sha256=9eIcOIUbaodNX1w9Tj2wl4U4wUmr5kF1R0i01fKUzGs,2974
+dataset/model_backwards_compatibility.py,sha256=69RKZwOrSyaBQvMCjOZiM-S-clVQu8cIKOUGGpI_87Y,19171
+dataset/model_validation.py,sha256=uj98wiz9SWbJc_He3kGGejy4JIIXM6RKaSccJfmo6wc,6672
+dataset/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dataset/statistic_subject_mapping.py,sha256=aOKQLvVNF4XWqVrDXVFUz-Hj_me6JeWW_uEPAKJvVJk,6145
+dataset/user_info.py,sha256=42PikdAQzC1FLOISC49yZO0IgVMWIq_QgxVD0xixaes,2541
+dataset/utility/__init__.py,sha256=pp6tUcgUbo8iq9OPtFKQrTbLuI3uY7NHptwWSTpasOU,33
+dataset/utility/constants.py,sha256=Wv1LIqq2P7ow6sToNdrTOAIMqvyPxNS2j6ArIB-GMds,2301
+dataset/utility/enums.py,sha256=C-qlB9ZI4Oy3q1ehbuF0GD7lqJJbuaspY_e8BDFu5DU,727
+dataset/utility/utils.py,sha256=j2A6DOgb4MmKaEGd5qW8DHxUsTZrZFLLAsvPW1BQIc0,14269
+dapla_toolbelt_metadata-0.1.1.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
+dapla_toolbelt_metadata-0.1.1.dist-info/METADATA,sha256=nmA0eQkasfLsUfoRtwKJBd9vD9m8mR9fC9z-AjZj-lM,5158
+dapla_toolbelt_metadata-0.1.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+dapla_toolbelt_metadata-0.1.1.dist-info/RECORD,,

dapla_toolbelt_metadata-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: true
+Tag: py3-none-any

dataset/__init__.py

@@ -0,0 +1,11 @@
+"""Document dataset."""
+
+from datadoc_model import model
+
+from .core import Datadoc
+from .dapla_dataset_path_info import DaplaDatasetPathInfo
+from .model_validation import ObligatoryDatasetWarning
+from .model_validation import ObligatoryVariableWarning
+from .utility.enums import DaplaRegion
+from .utility.enums import DaplaService
+from .utility.enums import SupportedLanguages

dataset/code_list.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from dataset.external_sources.external_sources import GetExternalSource
+from dataset.utility.enums import SupportedLanguages
+
+if TYPE_CHECKING:
+    from concurrent.futures import ThreadPoolExecutor
+
+    import pandas as pd
+from klass.classes.classification import KlassClassification
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class CodeListItem:
+    """Data structure for a code list item.
+
+    Attributes:
+        titles: A dictionary mapping language codes to titles.
+        code: The code associated with the item.
+    """
+
+    titles: dict[SupportedLanguages, str]
+    code: str
+
+    def get_title(self, language: SupportedLanguages) -> str:
+        """Return the title in the specified language.
+
+        Args:
+            language: The language code for which to get the title.
+
+        Returns:
+            The title in the specified language. It returns the title in Norwegian
+            Bokmål ("nb") if the language is either Norwegian Bokmål or Norwegian
+            Nynorsk, otherwise it returns the title in English ("en"). If none of
+            these are available, it returns an empty string and logs an exception.
+        """
+        try:
+            return self.titles[language]
+        except KeyError:
+            try:
+                return self.titles[
+                    (
+                        SupportedLanguages.NORSK_BOKMÅL
+                        if language
+                        in [
+                            SupportedLanguages.NORSK_BOKMÅL,
+                            SupportedLanguages.NORSK_NYNORSK,
+                        ]
+                        else SupportedLanguages.ENGLISH
+                    )
+                ]
+            except KeyError:
+                logger.exception(
+                    "Could not find title for subject %s  and language: %s",
+                    self,
+                    language.name,
+                )
+                return ""
+
+
+class CodeList(GetExternalSource):
+    """Class for retrieving classifications from Klass.
+
+    This class fetches a classification given a classification ID
+    and supports multiple languages.
+
+    Attributes:
+        supported_languages: A list of supported language codes.
+        _classifications: A list to store classification items.
+        classification_id: The ID of the classification to retrieve.
+        classifications_dataframes: A dictionary to store dataframes of
+            classifications.
+    """
+
+    def __init__(
+        self,
+        executor: ThreadPoolExecutor,
+        classification_id: int | None,
+    ) -> None:
+        """Initialize the CodeList with the given classification ID and executor.
+
+        Args:
+            executor: An instance of ThreadPoolExecutor to manage the asynchronous
+                execution of data fetching.
+            classification_id: The ID of the classification to retrieve.
+        """
+        self.supported_languages = [
+            SupportedLanguages.NORSK_BOKMÅL,
+            SupportedLanguages.ENGLISH,
+        ]
+        self._classifications: list[CodeListItem] = []
+        self.classification_id = classification_id
+        self.classifications_dataframes: (
+            dict[SupportedLanguages, pd.DataFrame] | None
+        ) = None
+        super().__init__(executor)
+
+    def _fetch_data_from_external_source(
+        self,
+    ) -> dict[SupportedLanguages, pd.DataFrame] | None:
+        """Fetch the classifications from Klass by classification ID.
+
+        This method retrieves classification data for each supported language and
+        stores it in a dictionary where the keys are language codes and the values
+        are pandas DataFrames containing the classification data.
+
+        Returns:
+            A dictionary mapping language codes to pandas DataFrames containing the
+            classification data for the given classification ID.
+            If an exception occurs during the fetching process, logs the exception
+            and returns None.
+        """
+        classifications_dataframes: dict[SupportedLanguages, pd.DataFrame] = {}
+        for i in self.supported_languages:
+            try:
+                classifications_dataframes[i] = (
+                    KlassClassification(
+                        str(self.classification_id),
+                        i,
+                    )
+                    .get_codes()
+                    .data
+                )
+            except Exception:  # noqa: PERF203
+                logger.exception(
+                    "Exception while getting classifications from Klass",
+                )
+                return None
+            else:
+                return classifications_dataframes
+        return None
+
+    def _extract_titles(
+        self,
+        dataframes: dict[SupportedLanguages, pd.DataFrame],
+    ) -> list[dict[SupportedLanguages, str]]:
+        """Extract titles from the dataframes for each supported language.
+
+        This method processes the provided dataframes and extracts the title from
+        each row for all supported languages, creating a list of dictionaries where
+        each dictionary maps language codes to titles.
+
+        Args:
+            dataframes: A dictionary mapping language codes to pandas DataFrames
+                containing classification data.
+
+        Returns:
+            A list of dictionaries, each mapping language codes to titles.
+            If a title is not available in a dataframe, the corresponding dictionary
+            value will be None.
+        """
+        list_of_titles = []
+        languages = list(dataframes)
+        for i in range(len(dataframes[SupportedLanguages.NORSK_BOKMÅL])):
+            titles = {}
+            for j in languages:
+                if "name" in dataframes[j]:
+                    titles[j] = dataframes[j].loc[:, "name"][i]
+                else:
+                    titles[j] = None
+            list_of_titles.append(titles)
+        return list_of_titles
+
+    def _create_code_list_from_dataframe(
+        self,
+        classifications_dataframes: dict[SupportedLanguages, pd.DataFrame],
+    ) -> list[CodeListItem]:
+        """Create a list of CodeListItem objects from the classification dataframes.
+
+        This method extracts titles from the provided dataframes and pairs them
+        with their corresponding classification codes to create a list of
+        CodeListItem objects.
+
+        Args:
+            classifications_dataframes: A dictionary mapping language codes to
+                pandas DataFrames containing classification data.
+
+        Returns:
+            A list of CodeListItem objects containing classification titles
+            and codes.
+        """
+        classification_names = self._extract_titles(classifications_dataframes)
+        classification_codes: list
+        if "code" in classifications_dataframes[SupportedLanguages.NORSK_BOKMÅL]:
+            classification_codes = (
+                classifications_dataframes[SupportedLanguages.NORSK_BOKMÅL]
+                .loc[:, "code"]
+                .to_list()
+            )
+        else:
+            classification_codes = [None] * len(classification_names)
+        classification_items = []
+        for a, b in zip(classification_names, classification_codes, strict=False):
+            classification_items.append(
+                CodeListItem(a, b),
+            )
+
+        return classification_items
+
+    def _get_classification_dataframe_if_loaded(self) -> bool:
+        """Check if the classification data from Klass is loaded.
+
+        This method verifies whether the classification data has been loaded.
+        If not, it retrieves the data from an external source and populates the
+        classifications. It logs the process and returns a boolean indicating the
+        success of the operation.
+
+        Returns:
+            True if the data is loaded and classifications are successfully extracted,
+            False otherwise.
+        """
+        if not self._classifications:
+            self.classifications_dataframes = self.retrieve_external_data()
+            if self.classifications_dataframes is not None:
+                self._classifications = self._create_code_list_from_dataframe(
+                    self.classifications_dataframes,
+                )
+                logger.debug(
+                    "Thread finished. found %s classifications",
+                    len(self._classifications),
+                )
+                return True
+            logger.warning(
+                "Thread is not done. Cannot get classifications from the dataframe.",
+            )
+        return False
+
+    @property
+    def classifications(self) -> list[CodeListItem]:
+        """Get the list of classifications.
+
+        Returns:
+            A list of CodeListItem objects.
+        """
+        self._get_classification_dataframe_if_loaded()
+
+        logger.debug("Got %s classifications subjects", len(self._classifications))
+        return self._classifications

dataset/config.py

@@ -0,0 +1,151 @@
+"""Configuration management for dataset package."""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from pprint import pformat
+from typing import Literal
+
+from dotenv import dotenv_values
+from dotenv import load_dotenv
+
+from dataset.utility.enums import DaplaRegion
+from dataset.utility.enums import DaplaService
+
+logging.basicConfig(level=logging.DEBUG, force=True)
+
+logger = logging.getLogger(__name__)
+
+DOT_ENV_FILE_PATH = Path(__file__).parent.joinpath(".env")
+
+JUPYTERHUB_USER = "JUPYTERHUB_USER"
+DAPLA_REGION = "DAPLA_REGION"
+DAPLA_SERVICE = "DAPLA_SERVICE"
+
+env_loaded = False
+
+
+def _load_dotenv_file() -> None:
+    global env_loaded  # noqa: PLW0603
+    if not env_loaded and DOT_ENV_FILE_PATH.exists():
+        load_dotenv(DOT_ENV_FILE_PATH)
+        env_loaded = True
+        logger.info(
+            "Loaded .env file with config keys: \n%s",
+            pformat(list(dotenv_values(DOT_ENV_FILE_PATH).keys())),
+        )
+
+
+def _get_config_item(item: str) -> str | None:
+    """Get a config item. Makes sure all access is logged."""
+    _load_dotenv_file()
+    value = os.getenv(item)
+    logger.debug("Config accessed. %s", item)
+    return value
+
+
+def get_jupyterhub_user() -> str | None:
+    """Get the JupyterHub user name."""
+    return _get_config_item(JUPYTERHUB_USER)
+
+
+def get_datadoc_dataset_path() -> str | None:
+    """Get the path to the dataset."""
+    return _get_config_item("DATADOC_DATASET_PATH")
+
+
+def get_log_level() -> int:
+    """Get the log level."""
+    # Magic numbers as defined in Python's stdlib logging
+    log_levels: dict[str, int] = {
+        "CRITICAL": 50,
+        "ERROR": 40,
+        "WARNING": 30,
+        "INFO": 20,
+        "DEBUG": 10,
+    }
+    if level_string := _get_config_item("DATADOC_LOG_LEVEL"):
+        try:
+            return log_levels[level_string.upper()]
+        except KeyError:
+            return log_levels["INFO"]
+    else:
+        return log_levels["INFO"]
+
+
+def get_log_formatter() -> Literal["simple", "json"]:
+    """Get log formatter configuration."""
+    if (
+        _get_config_item("DATADOC_ENABLE_JSON_FORMATTING") == "True"
+        or get_dapla_region() is not None
+    ):
+        return "json"
+    return "simple"
+
+
+def get_jupyterhub_service_prefix() -> str | None:
+    """Get the JupyterHub service prefix."""
+    return _get_config_item("JUPYTERHUB_SERVICE_PREFIX")
+
+
+def get_app_name() -> str:
+    """Get the name of the app. Defaults to 'Datadoc'."""
+    return _get_config_item("DATADOC_APP_NAME") or "Datadoc"
+
+
+def get_jupyterhub_http_referrer() -> str | None:
+    """Get the JupyterHub http referrer."""
+    return _get_config_item("JUPYTERHUB_HTTP_REFERER")
+
+
+def get_port() -> int:
+    """Get the port to run the app on."""
+    return int(_get_config_item("DATADOC_PORT") or 7002)
+
+
+def get_statistical_subject_source_url() -> str | None:
+    """Get the URL to the statistical subject source."""
+    return _get_config_item("DATADOC_STATISTICAL_SUBJECT_SOURCE_URL")
+
+
+def get_dapla_region() -> DaplaRegion | None:
+    """Get the Dapla region we're running on."""
+    if region := _get_config_item(DAPLA_REGION):
+        return DaplaRegion(region)
+
+    return None
+
+
+def get_dapla_service() -> DaplaService | None:
+    """Get the Dapla service we're running on."""
+    if service := _get_config_item(DAPLA_SERVICE):
+        return DaplaService(service)
+
+    return None
+
+
+def get_oidc_token() -> str | None:
+    """Get the JWT token from the environment."""
+    return _get_config_item("OIDC_TOKEN")
+
+
+def get_unit_code() -> int | None:
+    """The code for the Unit Type code list in Klass."""
+    return int(_get_config_item("DATADOC_UNIT_CODE") or 702)
+
+
+def get_measurement_unit_code() -> int | None:
+    """The code for the Measurement Unit code list in Klass."""
+    return int(_get_config_item("DATADOC_MEASUREMENT_UNIT") or 303)
+
+
+def get_organisational_unit_code() -> int | None:
+    """The code for the organisational units code list in Klass."""
+    return int(_get_config_item("DATADOC_ORGANISATIONAL_UNIT_CODE") or 83)
+
+
+def get_data_source_code() -> int | None:
+    """The code for the organisational units code list in Klass."""
+    return int(_get_config_item("DATADOC_DATA_SOURCE_CODE") or 712)