dapla-toolbelt-metadata 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

dapla_metadata/__init__.py

@@ -0,0 +1,3 @@
+"""Tools and clients for working with the Dapla Metadata system."""
+
+from . import datasets

{dataset → dapla_metadata/datasets}/__init__.py

@@ -6,6 +6,4 @@ from .core import Datadoc
 from .dapla_dataset_path_info import DaplaDatasetPathInfo
 from .model_validation import ObligatoryDatasetWarning
 from .model_validation import ObligatoryVariableWarning
-from .utility.enums import DaplaRegion
-from .utility.enums import DaplaService
-from .utility.enums import SupportedLanguages
+from .utility import enums

{dataset → dapla_metadata/datasets}/code_list.py

@@ -4,8 +4,8 @@ import logging
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
-from dataset.external_sources.external_sources import GetExternalSource
-from dataset.utility.enums import SupportedLanguages
+from dapla_metadata.datasets.external_sources.external_sources import GetExternalSource
+from dapla_metadata.datasets.utility.enums import SupportedLanguages
 
 if TYPE_CHECKING:
     from concurrent.futures import ThreadPoolExecutor

dapla_metadata/datasets/config.py

@@ -0,0 +1,82 @@
+"""Configuration management for dataset package."""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from pprint import pformat
+
+from dotenv import dotenv_values
+from dotenv import load_dotenv
+
+from dapla_metadata.datasets.utility.constants import (
+    DATADOC_STATISTICAL_SUBJECT_SOURCE_URL,
+)
+from dapla_metadata.datasets.utility.enums import DaplaRegion
+from dapla_metadata.datasets.utility.enums import DaplaService
+
+logging.basicConfig(level=logging.DEBUG, force=True)
+
+logger = logging.getLogger(__name__)
+
+DOT_ENV_FILE_PATH = Path(__file__).parent.joinpath(".env")
+
+JUPYTERHUB_USER = "JUPYTERHUB_USER"
+DAPLA_REGION = "DAPLA_REGION"
+DAPLA_SERVICE = "DAPLA_SERVICE"
+
+env_loaded = False
+
+
+def _load_dotenv_file() -> None:
+    global env_loaded  # noqa: PLW0603
+    if not env_loaded and DOT_ENV_FILE_PATH.exists():
+        load_dotenv(DOT_ENV_FILE_PATH)
+        env_loaded = True
+        logger.info(
+            "Loaded .env file with config keys: \n%s",
+            pformat(list(dotenv_values(DOT_ENV_FILE_PATH).keys())),
+        )
+
+
+def _get_config_item(item: str) -> str | None:
+    """Get a config item. Makes sure all access is logged."""
+    _load_dotenv_file()
+    value = os.getenv(item)
+    logger.debug("Config accessed. %s", item)
+    return value
+
+
+def get_jupyterhub_user() -> str | None:
+    """Get the JupyterHub user name."""
+    return _get_config_item(JUPYTERHUB_USER)
+
+
+def get_statistical_subject_source_url() -> str | None:
+    """Get the URL to the statistical subject source."""
+    return (
+        _get_config_item("DATADOC_STATISTICAL_SUBJECT_SOURCE_URL")
+        or DATADOC_STATISTICAL_SUBJECT_SOURCE_URL
+    )
+
+
+def get_dapla_region() -> DaplaRegion | None:
+    """Get the Dapla region we're running on."""
+    if region := _get_config_item(DAPLA_REGION):
+        return DaplaRegion(region)
+
+    return None
+
+
+def get_dapla_service() -> DaplaService | None:
+    """Get the Dapla service we're running on."""
+    if service := _get_config_item(DAPLA_SERVICE):
+        return DaplaService(service)
+
+    return None
+
+
+def get_oidc_token() -> str | None:
+    """Get the JWT token from the environment."""
+    return _get_config_item("OIDC_TOKEN")

{dataset → dapla_metadata/datasets}/core.py

@@ -13,28 +13,37 @@ from typing import TYPE_CHECKING
 from datadoc_model import model
 from datadoc_model.model import DataSetStatus
 
-from dataset import config
-from dataset import user_info
-from dataset.dapla_dataset_path_info import DaplaDatasetPathInfo
-from dataset.dataset_parser import DatasetParser
-from dataset.model_backwards_compatibility import is_metadata_in_container_structure
-from dataset.model_backwards_compatibility import upgrade_metadata
-from dataset.model_validation import ValidateDatadocMetadata
-from dataset.statistic_subject_mapping import StatisticSubjectMapping
-from dataset.utility.constants import DATASET_FIELDS_FROM_EXISTING_METADATA
-from dataset.utility.constants import DEFAULT_SPATIAL_COVERAGE_DESCRIPTION
-from dataset.utility.constants import INCONSISTENCIES_MESSAGE
-from dataset.utility.constants import METADATA_DOCUMENT_FILE_SUFFIX
-from dataset.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
-from dataset.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
-from dataset.utility.utils import calculate_percentage
-from dataset.utility.utils import derive_assessment_from_state
-from dataset.utility.utils import get_timestamp_now
-from dataset.utility.utils import normalize_path
-from dataset.utility.utils import num_obligatory_dataset_fields_completed
-from dataset.utility.utils import num_obligatory_variables_fields_completed
-from dataset.utility.utils import set_default_values_dataset
-from dataset.utility.utils import set_default_values_variables
+from dapla_metadata.datasets import config
+from dapla_metadata.datasets import user_info
+from dapla_metadata.datasets.dapla_dataset_path_info import DaplaDatasetPathInfo
+from dapla_metadata.datasets.dataset_parser import DatasetParser
+from dapla_metadata.datasets.model_backwards_compatibility import (
+    is_metadata_in_container_structure,
+)
+from dapla_metadata.datasets.model_backwards_compatibility import upgrade_metadata
+from dapla_metadata.datasets.model_validation import ValidateDatadocMetadata
+from dapla_metadata.datasets.statistic_subject_mapping import StatisticSubjectMapping
+from dapla_metadata.datasets.utility.constants import (
+    DEFAULT_SPATIAL_COVERAGE_DESCRIPTION,
+)
+from dapla_metadata.datasets.utility.constants import INCONSISTENCIES_MESSAGE
+from dapla_metadata.datasets.utility.constants import METADATA_DOCUMENT_FILE_SUFFIX
+from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
+from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
+from dapla_metadata.datasets.utility.utils import calculate_percentage
+from dapla_metadata.datasets.utility.utils import derive_assessment_from_state
+from dapla_metadata.datasets.utility.utils import get_timestamp_now
+from dapla_metadata.datasets.utility.utils import merge_variables
+from dapla_metadata.datasets.utility.utils import normalize_path
+from dapla_metadata.datasets.utility.utils import (
+    num_obligatory_dataset_fields_completed,
+)
+from dapla_metadata.datasets.utility.utils import (
+    num_obligatory_variables_fields_completed,
+)
+from dapla_metadata.datasets.utility.utils import override_dataset_fields
+from dapla_metadata.datasets.utility.utils import set_default_values_dataset
+from dapla_metadata.datasets.utility.utils import set_default_values_variables
 
 if TYPE_CHECKING:
     import pathlib
@@ -138,10 +147,11 @@ class Datadoc:
         """
         extracted_metadata: model.DatadocMetadata | None = None
         existing_metadata: model.DatadocMetadata | None = None
-        if self.metadata_document is not None and self.metadata_document.exists():
+        if self.metadata_document and self.metadata_document.exists():
             existing_metadata = self._extract_metadata_from_existing_document(
                 self.metadata_document,
             )
+
         if (
             self.dataset_path is not None
             and self.dataset == model.Dataset()
@@ -157,14 +167,7 @@ class Datadoc:
             and extracted_metadata is not None
             and existing_metadata is not None
         ):
-            if (
-                extracted_metadata.dataset is not None
-                and extracted_metadata.dataset.file_path is not None
-            ):
-                existing_file_path = extracted_metadata.dataset.file_path
-            else:
-                msg = "Could not access existing dataset file path"
-                raise ValueError(msg)
+            existing_file_path = self._get_existing_file_path(extracted_metadata)
             self._check_ready_to_merge(
                 self.dataset_path,
                 Path(existing_file_path),
@@ -181,31 +184,39 @@ class Datadoc:
             self.metadata_document = self.build_metadata_document_path(
                 self.dataset_path,
             )
-            if merged_metadata.dataset and merged_metadata.variables:
-                self.dataset = merged_metadata.dataset
-                self.variables = merged_metadata.variables
-            else:
-                msg = "Could not read metadata"
-                raise ValueError(msg)
-        elif (
-            existing_metadata
-            and existing_metadata.dataset
-            and existing_metadata.variables
+            self._set_metadata(merged_metadata)
+        else:
+            self._set_metadata(existing_metadata or extracted_metadata)
+        set_default_values_variables(self.variables)
+        set_default_values_dataset(self.dataset)
+        self._create_variables_lookup()
+
+    def _get_existing_file_path(
+        self,
+        extracted_metadata: model.DatadocMetadata | None,
+    ) -> str:
+        if (
+            extracted_metadata is not None
+            and extracted_metadata.dataset is not None
+            and extracted_metadata.dataset.file_path is not None
         ):
-            self.dataset = existing_metadata.dataset
-            self.variables = existing_metadata.variables
-        elif (
-            extracted_metadata
-            and extracted_metadata.dataset
-            and extracted_metadata.variables
+            return extracted_metadata.dataset.file_path
+        msg = "Could not access existing dataset file path"
+        raise ValueError(msg)
+
+    def _set_metadata(
+        self,
+        merged_metadata: model.DatadocMetadata | None,
+    ) -> None:
+        if not merged_metadata or not (
+            merged_metadata.dataset and merged_metadata.variables
         ):
-            self.dataset = extracted_metadata.dataset
-            self.variables = extracted_metadata.variables
-        else:
             msg = "Could not read metadata"
             raise ValueError(msg)
-        set_default_values_variables(self.variables)
-        set_default_values_dataset(self.dataset)
+        self.dataset = merged_metadata.dataset
+        self.variables = merged_metadata.variables
+
+    def _create_variables_lookup(self) -> None:
         self.variables_lookup = {
             v.short_name: v for v in self.variables if v.short_name
         }
@@ -300,55 +311,25 @@ class Datadoc:
                 "No existing metadata found, no merge to perform. Continuing with extracted metadata.",
             )
             return extracted_metadata or model.DatadocMetadata()
+
         if not extracted_metadata:
             return existing_metadata
+
         # Use the extracted metadata as a base
         merged_metadata = model.DatadocMetadata(
             dataset=copy.deepcopy(extracted_metadata.dataset),
             variables=[],
         )
-        if (
-            merged_metadata.dataset is not None
-            and existing_metadata.dataset is not None
-        ):
-            # Override the fields as defined
-            for field in DATASET_FIELDS_FROM_EXISTING_METADATA:
-                setattr(
-                    merged_metadata.dataset,
-                    field,
-                    getattr(existing_metadata.dataset, field),
-                )
+
+        override_dataset_fields(merged_metadata, extracted_metadata)
 
         # Merge variables.
         # For each extracted variable, copy existing metadata into the merged metadata
-        if (
-            existing_metadata.variables is not None
-            and extracted_metadata is not None
-            and extracted_metadata.variables is not None
-            and merged_metadata.variables is not None
-        ):
-            for extracted in extracted_metadata.variables:
-                existing = next(
-                    (
-                        existing
-                        for existing in existing_metadata.variables
-                        if existing.short_name == extracted.short_name
-                    ),
-                    None,
-                )
-                if existing:
-                    existing.id = None  # Set to None so that it will be set assigned a fresh ID later
-                    existing.contains_data_from = (
-                        extracted.contains_data_from or existing.contains_data_from
-                    )
-                    existing.contains_data_until = (
-                        extracted.contains_data_until or existing.contains_data_until
-                    )
-                    merged_metadata.variables.append(existing)
-                else:
-                    # If there is no existing metadata for this variable, we just use what we have extracted
-                    merged_metadata.variables.append(extracted)
-        return merged_metadata
+        return merge_variables(
+            existing_metadata,
+            extracted_metadata,
+            merged_metadata,
+        )
 
     def _extract_metadata_from_existing_document(
         self,

{dataset → dapla_metadata/datasets}/dataset_parser.py

@@ -19,7 +19,7 @@ from datadoc_model.model import LanguageStringTypeItem
 from datadoc_model.model import Variable
 from pyarrow import parquet as pq
 
-from dataset.utility.enums import SupportedLanguages
+from dapla_metadata.datasets.utility.enums import SupportedLanguages
 
 if TYPE_CHECKING:
     import pyarrow as pa

{dataset → dapla_metadata/datasets}/model_validation.py

@@ -11,17 +11,23 @@ from datadoc_model import model
 from pydantic import model_validator
 from typing_extensions import Self
 
-from dataset.utility.constants import DATE_VALIDATION_MESSAGE
-from dataset.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
-from dataset.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
-from dataset.utility.constants import OBLIGATORY_METADATA_WARNING
-from dataset.utility.utils import get_missing_obligatory_dataset_fields
-from dataset.utility.utils import get_missing_obligatory_variables_fields
-from dataset.utility.utils import get_timestamp_now
-from dataset.utility.utils import incorrect_date_order
-from dataset.utility.utils import num_obligatory_dataset_fields_completed
-from dataset.utility.utils import num_obligatory_variables_fields_completed
-from dataset.utility.utils import set_variables_inherit_from_dataset
+from dapla_metadata.datasets.utility.constants import DATE_VALIDATION_MESSAGE
+from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_DATASET_FIELDS
+from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
+from dapla_metadata.datasets.utility.constants import OBLIGATORY_METADATA_WARNING
+from dapla_metadata.datasets.utility.utils import get_missing_obligatory_dataset_fields
+from dapla_metadata.datasets.utility.utils import (
+    get_missing_obligatory_variables_fields,
+)
+from dapla_metadata.datasets.utility.utils import get_timestamp_now
+from dapla_metadata.datasets.utility.utils import incorrect_date_order
+from dapla_metadata.datasets.utility.utils import (
+    num_obligatory_dataset_fields_completed,
+)
+from dapla_metadata.datasets.utility.utils import (
+    num_obligatory_variables_fields_completed,
+)
+from dapla_metadata.datasets.utility.utils import set_variables_inherit_from_dataset
 
 if TYPE_CHECKING:
     from datetime import datetime

{dataset → dapla_metadata/datasets}/statistic_subject_mapping.py

@@ -9,8 +9,8 @@ import requests
 from bs4 import BeautifulSoup
 from bs4 import ResultSet
 
-from dataset.external_sources.external_sources import GetExternalSource
-from dataset.utility.enums import SupportedLanguages
+from dapla_metadata.datasets.external_sources.external_sources import GetExternalSource
+from dapla_metadata.datasets.utility.enums import SupportedLanguages
 
 if TYPE_CHECKING:
     from concurrent.futures import ThreadPoolExecutor
@@ -116,17 +116,18 @@ class StatisticSubjectMapping(GetExternalSource):
 
         Returns a BeautifulSoup ResultSet.
         """
+        if not self.source_url:
+            logger.debug("No statistic subject url supplied")
+            return None
+
         try:
-            url = str(self.source_url)
-            response = requests.get(url, timeout=30)
+            response = requests.get(str(self.source_url), timeout=30)
             response.encoding = "utf-8"
-            logger.debug("Got response %s from %s", response, url)
+            logger.debug("Got response %s from %s", response, self.source_url)
             soup = BeautifulSoup(response.text, features="xml")
             return soup.find_all("hovedemne")
         except requests.exceptions.RequestException:
-            logger.exception(
-                "Exception while fetching statistical structure ",
-            )
+            logger.exception("Exception while fetching statistical structure")
             return None
 
     def _parse_statistic_subject_structure_xml(

{dataset → dapla_metadata/datasets}/user_info.py

@@ -6,9 +6,9 @@ from typing import Protocol
 
 import jwt
 
-from dataset import config
-from dataset.utility.enums import DaplaRegion
-from dataset.utility.enums import DaplaService
+from dapla_metadata.datasets import config
+from dapla_metadata.datasets.utility.enums import DaplaRegion
+from dapla_metadata.datasets.utility.enums import DaplaService
 
 logger = logging.getLogger(__name__)
 

{dataset → dapla_metadata/datasets}/utility/constants.py

@@ -90,3 +90,7 @@ DATASET_FIELDS_FROM_EXISTING_METADATA = [
 ]
 
 METADATA_DOCUMENT_FILE_SUFFIX = "__DOC.json"
+
+DATADOC_STATISTICAL_SUBJECT_SOURCE_URL = (
+    "https://www.ssb.no/xp/_/service/mimir/subjectStructurStatistics"
+)

{dataset → dapla_metadata/datasets}/utility/utils.py

@@ -14,13 +14,20 @@ from datadoc_model.model import Assessment
 from datadoc_model.model import DataSetState
 from datadoc_model.model import VariableRole
 
-from dataset.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
-from dataset.utility.constants import OBLIGATORY_DATASET_METADATA_IDENTIFIERS
-from dataset.utility.constants import (
+from dapla_metadata.datasets.utility.constants import (
+    DATASET_FIELDS_FROM_EXISTING_METADATA,
+)
+from dapla_metadata.datasets.utility.constants import NUM_OBLIGATORY_VARIABLES_FIELDS
+from dapla_metadata.datasets.utility.constants import (
+    OBLIGATORY_DATASET_METADATA_IDENTIFIERS,
+)
+from dapla_metadata.datasets.utility.constants import (
     OBLIGATORY_DATASET_METADATA_IDENTIFIERS_MULTILANGUAGE,
 )
-from dataset.utility.constants import OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS
-from dataset.utility.constants import (
+from dapla_metadata.datasets.utility.constants import (
+    OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS,
+)
+from dapla_metadata.datasets.utility.constants import (
     OBLIGATORY_VARIABLES_METADATA_IDENTIFIERS_MULTILANGUAGE,
 )
 
@@ -403,3 +410,83 @@ def running_in_notebook() -> bool:
         # interpreters and will throw a NameError. Therefore we're not running
         # in Jupyter.
         return False
+
+
+def override_dataset_fields(
+    merged_metadata: model.DatadocMetadata,
+    existing_metadata: model.DatadocMetadata,
+) -> None:
+    """Overrides specific fields in the dataset of `merged_metadata` with values from the dataset of `existing_metadata`.
+
+    This function iterates over a predefined list of fields, `DATASET_FIELDS_FROM_EXISTING_METADATA`,
+    and sets the corresponding fields in the `merged_metadata.dataset` object to the values
+    from the `existing_metadata.dataset` object.
+
+    Args:
+        merged_metadata: An instance of `DatadocMetadata` containing the dataset to be updated.
+        existing_metadata: An instance of `DatadocMetadata` containing the dataset whose values are used to update `merged_metadata.dataset`.
+
+    Returns:
+        `None`.
+    """
+    if merged_metadata.dataset and existing_metadata.dataset:
+        # Override the fields as defined
+        for field in DATASET_FIELDS_FROM_EXISTING_METADATA:
+            setattr(
+                merged_metadata.dataset,
+                field,
+                getattr(existing_metadata.dataset, field),
+            )
+
+
+def merge_variables(
+    existing_metadata: model.DatadocMetadata,
+    extracted_metadata: model.DatadocMetadata,
+    merged_metadata: model.DatadocMetadata,
+) -> model.DatadocMetadata:
+    """Merges variables from the extracted metadata into the existing metadata and updates the merged metadata.
+
+    This function compares the variables from `extracted_metadata` with those in `existing_metadata`.
+    For each variable in `extracted_metadata`, it checks if a variable with the same `short_name` exists
+    in `existing_metadata`. If a match is found, it updates the existing variable with information from
+    `extracted_metadata`. If no match is found, the variable from `extracted_metadata` is directly added to `merged_metadata`.
+
+    Args:
+        existing_metadata: The metadata object containing the current state of variables.
+        extracted_metadata: The metadata object containing new or updated variables to merge.
+        merged_metadata: The metadata object that will contain the result of the merge.
+
+    Returns:
+        model.DatadocMetadata: The `merged_metadata` object containing variables from both `existing_metadata`
+        and `extracted_metadata`.
+    """
+    if (
+        existing_metadata.variables is not None
+        and extracted_metadata is not None
+        and extracted_metadata.variables is not None
+        and merged_metadata.variables is not None
+    ):
+        for extracted in extracted_metadata.variables:
+            existing = next(
+                (
+                    existing
+                    for existing in existing_metadata.variables
+                    if existing.short_name == extracted.short_name
+                ),
+                None,
+            )
+            if existing:
+                existing.id = (
+                    None  # Set to None so that it will be set assigned a fresh ID later
+                )
+                existing.contains_data_from = (
+                    extracted.contains_data_from or existing.contains_data_from
+                )
+                existing.contains_data_until = (
+                    extracted.contains_data_until or existing.contains_data_until
+                )
+                merged_metadata.variables.append(existing)
+            else:
+                # If there is no existing metadata for this variable, we just use what we have extracted
+                merged_metadata.variables.append(extracted)
+    return merged_metadata

{dapla_toolbelt_metadata-0.1.1.dist-info → dapla_toolbelt_metadata-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dapla-toolbelt-metadata
-Version: 0.1.1
+Version: 0.2.0
 Summary: Dapla Toolbelt Metadata
 Home-page: https://github.com/statisticsnorway/dapla-toolbelt-metadata
 License: MIT
@@ -15,30 +15,17 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: arrow (>=1.3.0)
 Requires-Dist: beautifulsoup4 (>=4.12.3)
-Requires-Dist: black (>=24.8.0,<25.0.0)
 Requires-Dist: bs4 (>=0.0.2,<0.0.3)
-Requires-Dist: click (>=8.0.1)
 Requires-Dist: cloudpathlib[gs] (>=0.17.0)
-Requires-Dist: coverage (>=7.6.1,<8.0.0)
 Requires-Dist: dapla-toolbelt (>=1.3.3)
-Requires-Dist: faker (>=26.1.0,<27.0.0)
-Requires-Dist: furo (>=2024.7.18,<2025.0.0)
-Requires-Dist: gunicorn (>=21.2.0)
 Requires-Dist: pandas (>=1.4.2)
-Requires-Dist: pre-commit (>=3.8.0,<4.0.0)
 Requires-Dist: pyarrow (>=8.0.0)
 Requires-Dist: pydantic (>=2.5.2)
-Requires-Dist: pygments (>=2.18.0,<3.0.0)
 Requires-Dist: pyjwt (>=2.8.0)
-Requires-Dist: pytest (>=8.3.2,<9.0.0)
-Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
 Requires-Dist: python-dotenv (>=1.0.1)
 Requires-Dist: requests (>=2.31.0)
-Requires-Dist: requests-mock (>=1.12.1,<2.0.0)
-Requires-Dist: ruff (>=0.5.6,<0.6.0)
 Requires-Dist: ssb-datadoc-model (>=6.0.0,<7.0.0)
 Requires-Dist: ssb-klass-python (>=0.0.9)
-Requires-Dist: types-beautifulsoup4 (>=4.12.0.20240511,<5.0.0.0)
 Project-URL: Changelog, https://github.com/statisticsnorway/dapla-toolbelt-metadata/releases
 Project-URL: Documentation, https://statisticsnorway.github.io/dapla-toolbelt-metadata
 Project-URL: Repository, https://github.com/statisticsnorway/dapla-toolbelt-metadata
@@ -71,14 +58,16 @@ Description-Content-Type: text/markdown
 [black]: https://github.com/psf/black
 [poetry]: https://python-poetry.org/
 
+Tools and clients for working with the Dapla Metadata system.
+
 ## Features
 
-- TODO
+- Create and update metadata for datasets (Datadoc).
 
-## Requirements
+### Coming
 
-- TODO
-- Python
+- Read, create and update variable definitions.
+- Publish dataset metadata to Statistics Norway's data catalogue.
 
 ## Installation
 
@@ -90,7 +79,7 @@ pip install dapla-toolbelt-metadata
 
 ## Usage
 
-Please see the [Reference Guide] for details.
+Instructions and examples may be found in the [Dapla Manual](https://manual.dapla.ssb.no/statistikkere/). Please see the [Reference Guide] for API documentation.
 
 ## Contributing
 

dapla_toolbelt_metadata-0.2.0.dist-info/RECORD

@@ -0,0 +1,22 @@
+dapla_metadata/__init__.py,sha256=i8bhabWldn-kGfbdCMANY-iC0LbacRIPU1-LEorXqVY,92
+dapla_metadata/datasets/__init__.py,sha256=TvzskpdFC6hGcC9_55URT5jr5wNAPzXuISd2UjJWM_8,280
+dapla_metadata/datasets/code_list.py,sha256=kp1O6sUiUAP9WKlWY8IgHWx_1IOzJA63WveHqolgKmg,9082
+dapla_metadata/datasets/config.py,sha256=lmujwsKKT_X94AL1zHiot24TOyynm92N5dhMti-dhYs,2215
+dapla_metadata/datasets/core.py,sha256=gUt-LDoQnT8-sFvSpodC6qg4ALgITWkSNOn4dVZuuCQ,21461
+dapla_metadata/datasets/dapla_dataset_path_info.py,sha256=7wwVwykJUaRbqCZrAMsZsOd1p_xO8bHe5LhNOLE8j6k,21600
+dapla_metadata/datasets/dataset_parser.py,sha256=PhhGvkUrmQ2nypYgg4JWushb7gFvde65XbRhmXdPZUI,7965
+dapla_metadata/datasets/external_sources/__init__.py,sha256=qvIdXwqyEmXNUCB94ZtZXRzifdW4hiXASFFPtC70f6E,83
+dapla_metadata/datasets/external_sources/external_sources.py,sha256=9eIcOIUbaodNX1w9Tj2wl4U4wUmr5kF1R0i01fKUzGs,2974
+dapla_metadata/datasets/model_backwards_compatibility.py,sha256=69RKZwOrSyaBQvMCjOZiM-S-clVQu8cIKOUGGpI_87Y,19171
+dapla_metadata/datasets/model_validation.py,sha256=j0yHxiV3QusAwzohUG-pSq-6ojhynnto0z_dD5cSnVs,6875
+dapla_metadata/datasets/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dapla_metadata/datasets/statistic_subject_mapping.py,sha256=QdC22DUBOdRgsfmTTEUr9CKCE8fKjMq6-Ezdr6Eof_A,6254
+dapla_metadata/datasets/user_info.py,sha256=6cjIhYHUQmCKnhBjvH6GN3B6H3J-pOjc9NdQcs-NGsE,2589
+dapla_metadata/datasets/utility/__init__.py,sha256=pp6tUcgUbo8iq9OPtFKQrTbLuI3uY7NHptwWSTpasOU,33
+dapla_metadata/datasets/utility/constants.py,sha256=V9ixHTShK1uBcSesaVoKDSHzAh3CX1ATO-Z3wZHxAKs,2417
+dapla_metadata/datasets/utility/enums.py,sha256=C-qlB9ZI4Oy3q1ehbuF0GD7lqJJbuaspY_e8BDFu5DU,727
+dapla_metadata/datasets/utility/utils.py,sha256=wCRZBB3YwXTB_BaqDABaaeBjVmMZP8z3grjfTxX308k,18018
+dapla_toolbelt_metadata-0.2.0.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
+dapla_toolbelt_metadata-0.2.0.dist-info/METADATA,sha256=UswEf8UvTCen439oh3m6j-HI8EC8CJpCqThyJis_4no,4927
+dapla_toolbelt_metadata-0.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+dapla_toolbelt_metadata-0.2.0.dist-info/RECORD,,

dapla_toolbelt_metadata-0.1.1.dist-info/RECORD

@@ -1,21 +0,0 @@
-dataset/__init__.py,sha256=aa6dF2ddfeljw1SElaVqAF6YewuZ0Fxk3q7iebtWR2E,378
-dataset/code_list.py,sha256=SqzassTXL-Gr4TqomXStJFiI5gSN0lXjbmZLDJadMrQ,9050
-dataset/config.py,sha256=EWTmrkLWYKSHVrzEWQO16_CaWrxiUllAnRJFbvEoono,4239
-dataset/core.py,sha256=E7OmFM1iTKlKuvQXuwhucO3z5pKg8fTkE2TmOYFi7_M,22654
-dataset/dapla_dataset_path_info.py,sha256=7wwVwykJUaRbqCZrAMsZsOd1p_xO8bHe5LhNOLE8j6k,21600
-dataset/dataset_parser.py,sha256=AvN4cKaDvP4VwplNR5uvXJdiZh4ippNcFTBll-HhH-4,7949
-dataset/external_sources/__init__.py,sha256=qvIdXwqyEmXNUCB94ZtZXRzifdW4hiXASFFPtC70f6E,83
-dataset/external_sources/external_sources.py,sha256=9eIcOIUbaodNX1w9Tj2wl4U4wUmr5kF1R0i01fKUzGs,2974
-dataset/model_backwards_compatibility.py,sha256=69RKZwOrSyaBQvMCjOZiM-S-clVQu8cIKOUGGpI_87Y,19171
-dataset/model_validation.py,sha256=uj98wiz9SWbJc_He3kGGejy4JIIXM6RKaSccJfmo6wc,6672
-dataset/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dataset/statistic_subject_mapping.py,sha256=aOKQLvVNF4XWqVrDXVFUz-Hj_me6JeWW_uEPAKJvVJk,6145
-dataset/user_info.py,sha256=42PikdAQzC1FLOISC49yZO0IgVMWIq_QgxVD0xixaes,2541
-dataset/utility/__init__.py,sha256=pp6tUcgUbo8iq9OPtFKQrTbLuI3uY7NHptwWSTpasOU,33
-dataset/utility/constants.py,sha256=Wv1LIqq2P7ow6sToNdrTOAIMqvyPxNS2j6ArIB-GMds,2301
-dataset/utility/enums.py,sha256=C-qlB9ZI4Oy3q1ehbuF0GD7lqJJbuaspY_e8BDFu5DU,727
-dataset/utility/utils.py,sha256=j2A6DOgb4MmKaEGd5qW8DHxUsTZrZFLLAsvPW1BQIc0,14269
-dapla_toolbelt_metadata-0.1.1.dist-info/LICENSE,sha256=np3IfD5m0ZUofn_kVzDZqliozuiO6wrktw3LRPjyEiI,1073
-dapla_toolbelt_metadata-0.1.1.dist-info/METADATA,sha256=nmA0eQkasfLsUfoRtwKJBd9vD9m8mR9fC9z-AjZj-lM,5158
-dapla_toolbelt_metadata-0.1.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-dapla_toolbelt_metadata-0.1.1.dist-info/RECORD,,

dataset/config.py

@@ -1,151 +0,0 @@
-"""Configuration management for dataset package."""
-
-from __future__ import annotations
-
-import logging
-import os
-from pathlib import Path
-from pprint import pformat
-from typing import Literal
-
-from dotenv import dotenv_values
-from dotenv import load_dotenv
-
-from dataset.utility.enums import DaplaRegion
-from dataset.utility.enums import DaplaService
-
-logging.basicConfig(level=logging.DEBUG, force=True)
-
-logger = logging.getLogger(__name__)
-
-DOT_ENV_FILE_PATH = Path(__file__).parent.joinpath(".env")
-
-JUPYTERHUB_USER = "JUPYTERHUB_USER"
-DAPLA_REGION = "DAPLA_REGION"
-DAPLA_SERVICE = "DAPLA_SERVICE"
-
-env_loaded = False
-
-
-def _load_dotenv_file() -> None:
-    global env_loaded  # noqa: PLW0603
-    if not env_loaded and DOT_ENV_FILE_PATH.exists():
-        load_dotenv(DOT_ENV_FILE_PATH)
-        env_loaded = True
-        logger.info(
-            "Loaded .env file with config keys: \n%s",
-            pformat(list(dotenv_values(DOT_ENV_FILE_PATH).keys())),
-        )
-
-
-def _get_config_item(item: str) -> str | None:
-    """Get a config item. Makes sure all access is logged."""
-    _load_dotenv_file()
-    value = os.getenv(item)
-    logger.debug("Config accessed. %s", item)
-    return value
-
-
-def get_jupyterhub_user() -> str | None:
-    """Get the JupyterHub user name."""
-    return _get_config_item(JUPYTERHUB_USER)
-
-
-def get_datadoc_dataset_path() -> str | None:
-    """Get the path to the dataset."""
-    return _get_config_item("DATADOC_DATASET_PATH")
-
-
-def get_log_level() -> int:
-    """Get the log level."""
-    # Magic numbers as defined in Python's stdlib logging
-    log_levels: dict[str, int] = {
-        "CRITICAL": 50,
-        "ERROR": 40,
-        "WARNING": 30,
-        "INFO": 20,
-        "DEBUG": 10,
-    }
-    if level_string := _get_config_item("DATADOC_LOG_LEVEL"):
-        try:
-            return log_levels[level_string.upper()]
-        except KeyError:
-            return log_levels["INFO"]
-    else:
-        return log_levels["INFO"]
-
-
-def get_log_formatter() -> Literal["simple", "json"]:
-    """Get log formatter configuration."""
-    if (
-        _get_config_item("DATADOC_ENABLE_JSON_FORMATTING") == "True"
-        or get_dapla_region() is not None
-    ):
-        return "json"
-    return "simple"
-
-
-def get_jupyterhub_service_prefix() -> str | None:
-    """Get the JupyterHub service prefix."""
-    return _get_config_item("JUPYTERHUB_SERVICE_PREFIX")
-
-
-def get_app_name() -> str:
-    """Get the name of the app. Defaults to 'Datadoc'."""
-    return _get_config_item("DATADOC_APP_NAME") or "Datadoc"
-
-
-def get_jupyterhub_http_referrer() -> str | None:
-    """Get the JupyterHub http referrer."""
-    return _get_config_item("JUPYTERHUB_HTTP_REFERER")
-
-
-def get_port() -> int:
-    """Get the port to run the app on."""
-    return int(_get_config_item("DATADOC_PORT") or 7002)
-
-
-def get_statistical_subject_source_url() -> str | None:
-    """Get the URL to the statistical subject source."""
-    return _get_config_item("DATADOC_STATISTICAL_SUBJECT_SOURCE_URL")
-
-
-def get_dapla_region() -> DaplaRegion | None:
-    """Get the Dapla region we're running on."""
-    if region := _get_config_item(DAPLA_REGION):
-        return DaplaRegion(region)
-
-    return None
-
-
-def get_dapla_service() -> DaplaService | None:
-    """Get the Dapla service we're running on."""
-    if service := _get_config_item(DAPLA_SERVICE):
-        return DaplaService(service)
-
-    return None
-
-
-def get_oidc_token() -> str | None:
-    """Get the JWT token from the environment."""
-    return _get_config_item("OIDC_TOKEN")
-
-
-def get_unit_code() -> int | None:
-    """The code for the Unit Type code list in Klass."""
-    return int(_get_config_item("DATADOC_UNIT_CODE") or 702)
-
-
-def get_measurement_unit_code() -> int | None:
-    """The code for the Measurement Unit code list in Klass."""
-    return int(_get_config_item("DATADOC_MEASUREMENT_UNIT") or 303)
-
-
-def get_organisational_unit_code() -> int | None:
-    """The code for the organisational units code list in Klass."""
-    return int(_get_config_item("DATADOC_ORGANISATIONAL_UNIT_CODE") or 83)
-
-
-def get_data_source_code() -> int | None:
-    """The code for the organisational units code list in Klass."""
-    return int(_get_config_item("DATADOC_DATA_SOURCE_CODE") or 712)