statwrapper 0.1.0__py3-none-any.whl

statwrapper/__init__.py

@@ -0,0 +1,31 @@
+from .base_api_client import APIWrapper
+from .http import RateLimitedSession
+from .models import DiscoveredDataset, Provider, ResolvedDatasetMetadata
+from .provider_registry import (
+    create_wrapper,
+    get_provider,
+    get_wrapper_class,
+    load_providers,
+)
+from .statwrapper import (
+    StatWrapper,
+    discover_provider_datasets,
+    get_datasets,
+    resolve_dataset_metadata,
+)
+
+__all__ = [
+    "APIWrapper",
+    "DiscoveredDataset",
+    "Provider",
+    "RateLimitedSession",
+    "ResolvedDatasetMetadata",
+    "StatWrapper",
+    "create_wrapper",
+    "discover_provider_datasets",
+    "get_datasets",
+    "get_provider",
+    "get_wrapper_class",
+    "load_providers",
+    "resolve_dataset_metadata",
+]

statwrapper/api_clients/__init__.py

@@ -0,0 +1,11 @@
+from .dst_client import DstClient
+from .eurostat_client import EurostatClient
+from .pxweb2_client import PxWeb2Client
+from .pxweb_client import PxWebClient
+
+__all__ = [
+    "DstClient",
+    "EurostatClient",
+    "PxWebClient",
+    "PxWeb2Client",
+]

statwrapper/api_clients/dst_client.py

@@ -0,0 +1,249 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from ..base_api_client import APIWrapper
+from ..models import DiscoveredDataset, Provider, ResolvedDatasetMetadata
+from ..utils import (
+    detect_role,
+    determine_time_unit,
+    normalize_contact,
+    normalize_note,
+    parse_dt,
+)
+
+
+def _collect_tables(subjects_tree: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
+    tables_by_id: dict[str, dict[str, Any]] = {}
+
+    def visit(node: dict[str, Any]) -> None:
+        for table in node.get("tables") or []:
+            if not isinstance(table, dict):
+                continue
+            table_id = str(table.get("id") or "").strip()
+            if not table_id:
+                continue
+            existing = tables_by_id.setdefault(table_id, {"id": table_id})
+            for key, value in table.items():
+                if existing.get(key) in (None, "", []) and value not in (None, "", []):
+                    existing[key] = value
+        for child in node.get("subjects") or []:
+            if isinstance(child, dict):
+                visit(child)
+
+    for root in subjects_tree:
+        if isinstance(root, dict):
+            visit(root)
+    return tables_by_id
+
+
+def _parse_paths(subjects_tree: list[dict[str, Any]]) -> dict[str, list[list[dict[str, str]]]]:
+    code_to_paths: dict[str, list[list[dict[str, str]]]] = {}
+
+    def visit(node: dict[str, Any], path: list[dict[str, str]]) -> None:
+        node_id = str(node.get("id") or "").strip()
+        node_label = str(node.get("description") or node.get("text") or node_id).strip()
+        current_path = path + ([{"id": node_id, "label": node_label}] if node_id else [])
+        for table in node.get("tables") or []:
+            if not isinstance(table, dict):
+                continue
+            table_id = str(table.get("id") or "").strip()
+            if not table_id:
+                continue
+            code_to_paths.setdefault(table_id, []).append(
+                current_path + [{"id": table_id, "label": str(table.get("text") or table_id).strip()}]
+            )
+        for child in node.get("subjects") or []:
+            if isinstance(child, dict):
+                visit(child, current_path)
+
+    for root in subjects_tree:
+        if isinstance(root, dict):
+            visit(root, [])
+    return code_to_paths
+
+
+class DstClient(APIWrapper):
+    def __init__(self, provider: Provider, **kwargs: Any) -> None:
+        super().__init__(
+            provider_code=provider.provider_code,
+            label=provider.label,
+            language=kwargs.pop("language"),
+            json_request_handler=kwargs.pop("json_request_handler"),
+            text_request_handler=kwargs.pop("text_request_handler", None),
+            bytes_request_handler=kwargs.pop("bytes_request_handler", None),
+            logger=kwargs.pop("logger", None),
+        )
+        self.provider = provider
+        self.api_type = provider.api_type
+        self.base_api_url = provider.base_api_url or ""
+        self.base_web_url = provider.base_web_url
+
+    def _subjects_url(self) -> str:
+        return f"{self.base_api_url.rstrip('/')}/subjects"
+
+    def _tableinfo_url(self, dataset_code: str) -> str:
+        return f"{self.base_api_url.rstrip('/')}/tableinfo/{dataset_code}"
+
+    def _data_url(self, dataset_code: str) -> str:
+        return f"{self.base_api_url.rstrip('/')}/data/{dataset_code}/JSONSTAT"
+
+    async def health_check(self, dataset_code: str | None = None) -> bool:
+        url = self._subjects_url() if dataset_code is None else self._tableinfo_url(dataset_code)
+        payload = await self._get_json(url, params={"lang": self.language})
+        return payload is not None
+
+    async def discover_datasets(
+        self,
+        task_id: uuid.UUID,
+        **_: Any,
+    ) -> list[DiscoveredDataset]:
+        payload = await self._get_json(
+            self._subjects_url(),
+            params={
+                "lang": self.language,
+                "recursive": "true",
+                "omitSubjectsWithoutTables": "true",
+                "includeTables": "true",
+                "format": "JSON",
+            },
+        )
+        if not isinstance(payload, list):
+            return []
+        tables_by_id = _collect_tables(payload)
+        paths_by_id = _parse_paths(payload)
+        output: list[DiscoveredDataset] = []
+        for dataset_code, table in sorted(tables_by_id.items()):
+            paths = paths_by_id.get(dataset_code, [])
+            primary_path = paths[0] if paths else []
+            subject = primary_path[0] if primary_path else None
+            first_period = str(table.get("firstPeriod") or "").strip() or None
+            last_period = str(table.get("latestPeriod") or table.get("lastPeriod") or "").strip() or None
+            updated = parse_dt(table.get("updated")) or parse_dt("1970-01-01T00:00:00+00:00")
+            if updated is None:
+                continue
+            output.append(
+                DiscoveredDataset(
+                    task_id=task_id,
+                    provider_code=self.provider_code,
+                    dataset_code=dataset_code,
+                    language=self.language,
+                    updated=updated,
+                    label=str(table.get("text") or dataset_code).strip(),
+                    description=str(table.get("description")).strip() if table.get("description") is not None else None,
+                    source=f"Statistics Denmark - statbank.dk/{dataset_code}",
+                    subject_code=subject["id"] if subject else None,
+                    subject_label=subject["label"] if subject else None,
+                    time_unit=determine_time_unit(first_period, last_period) if first_period and last_period else None,
+                    first_period=first_period,
+                    last_period=last_period,
+                    discontinued=(not bool(table.get("active")) if isinstance(table.get("active"), bool) else False),
+                    metadata_url=f"{self._tableinfo_url(dataset_code)}?lang={self.language}",
+                    data_url=f"{self._data_url(dataset_code)}?lang={self.language}",
+                    web_url=f"{self.base_web_url.rstrip('/')}/{dataset_code}" if self.base_web_url else None,
+                    paths=paths or None,
+                    extension={"unit": table.get("unit"), "variable_names": table.get("variables")},
+                )
+            )
+        return output
+
+    async def resolve_dataset_metadata(
+        self,
+        discovered: DiscoveredDataset,
+        task_id: uuid.UUID | None = None,
+        **_: Any,
+    ) -> ResolvedDatasetMetadata:
+        payload = await self._get_json(
+            self._tableinfo_url(discovered.dataset_code),
+            params={"lang": self.language},
+        )
+        dimensions: dict[str, dict[str, Any]] = {}
+        dimension_ids: list[str] = []
+        required_dimensions: dict[str, bool | None] = {}
+        role_map: dict[str, list[str]] = {}
+        extension = dict(discovered.extension)
+        if isinstance(payload, dict):
+            for position, variable in enumerate(payload.get("variables") or []):
+                if not isinstance(variable, dict):
+                    continue
+                dimension_id = str(variable.get("id") or "").strip()
+                if not dimension_id:
+                    continue
+                values = variable.get("values") if isinstance(variable.get("values"), list) else []
+                index: dict[str, int] = {}
+                labels: dict[str, str] = {}
+                for ordinal, item in enumerate(values):
+                    if not isinstance(item, dict):
+                        continue
+                    value_id = str(item.get("id") or "").strip()
+                    if not value_id:
+                        continue
+                    index[value_id] = ordinal
+                    labels[value_id] = str(item.get("text") or value_id).strip()
+                if not index:
+                    continue
+                label = str(variable.get("text") or dimension_id).strip()
+                dimensions[dimension_id] = {
+                    "label": label,
+                    "category": {"index": index, "label": labels},
+                }
+                dimension_ids.append(dimension_id)
+                elimination = variable.get("elimination")
+                required_dimensions[dimension_id] = None if elimination is None else not bool(elimination)
+                role = "geo" if variable.get("map") else detect_role(dimension_id, label, bool(variable.get("time")))
+                if role:
+                    role_map.setdefault(role, []).append(dimension_id)
+                extension.setdefault("dimension_extensions", {})[dimension_id] = {
+                    "extension": {"position": position}
+                }
+        label = (
+            str(payload.get("text")).strip()
+            if isinstance(payload, dict) and payload.get("text") is not None
+            else (discovered.label or discovered.dataset_code)
+        )
+        description = (
+            str(payload.get("description")).strip()
+            if isinstance(payload, dict) and payload.get("description") is not None
+            else discovered.description
+        )
+        updated = (
+            parse_dt(payload.get("updated"))
+            if isinstance(payload, dict)
+            else None
+        ) or discovered.updated
+        return ResolvedDatasetMetadata(
+            task_id=task_id or discovered.task_id,
+            provider_code=discovered.provider_code,
+            dataset_code=discovered.dataset_code,
+            language=discovered.language,
+            updated=updated,
+            label=label,
+            time_unit=discovered.time_unit or "Other",
+            first_period=discovered.first_period or "",
+            last_period=discovered.last_period or discovered.first_period or "",
+            paths=discovered.paths or [],
+            role=role_map,
+            metadata_url=discovered.metadata_url or f"{self._tableinfo_url(discovered.dataset_code)}?lang={self.language}",
+            data_url=discovered.data_url or f"{self._data_url(discovered.dataset_code)}?lang={self.language}",
+            dimension=dimensions,
+            required_dimensions=required_dimensions,
+            note=(normalize_note(payload.get("footnote")) if isinstance(payload, dict) else None) or discovered.note,
+            source=discovered.source,
+            description=description,
+            discontinued=not bool(payload.get("active")) if isinstance(payload, dict) and isinstance(payload.get("active"), bool) else discovered.discontinued,
+            subject_code=discovered.subject_code,
+            subject_label=discovered.subject_label,
+            web_url=discovered.web_url,
+            doc_url=(
+                str(payload.get("documentation", {}).get("url")).strip()
+                if isinstance(payload, dict)
+                and isinstance(payload.get("documentation"), dict)
+                and payload["documentation"].get("url") is not None
+                else discovered.doc_url
+            ),
+            official_statistics=discovered.official_statistics,
+            contact=(normalize_contact(payload.get("contacts")) if isinstance(payload, dict) else None) or discovered.contact,
+            dimension_ids=dimension_ids or None,
+            extension=extension,
+        )

statwrapper/api_clients/eurostat_client.py

@@ -0,0 +1,359 @@
+from __future__ import annotations
+
+import asyncio
+import csv
+import gzip
+import io
+import re
+import uuid
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any
+
+from ..base_api_client import APIWrapper
+from ..models import DiscoveredDataset, Provider, ResolvedDatasetMetadata
+from ..utils import parse_dt
+
+DATA_URL_TEMPLATE = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/{dataset_code}"
+METADATA_URL_TEMPLATE = (
+    "https://ec.europa.eu/eurostat/api/dissemination/sdmx/3.0/data/dataflow/"
+    "ESTAT/{dataset_code}/1.0?format=JSON&lang={language}&compress=false&attributes=none&measures=none"
+)
+WEB_URL_TEMPLATE = "https://ec.europa.eu/eurostat/databrowser/product/page/{dataset_code}"
+DOC_URL_TEMPLATE = "https://ec.europa.eu/eurostat/cache/metadata/en/{lowercase_dataset_code}.htm"
+TOC_URL_TEMPLATE = "https://ec.europa.eu/eurostat/api/dissemination/catalogue/toc/txt?lang={language}"
+DATAFLOWS_URL = (
+    "https://ec.europa.eu/eurostat/api/dissemination/sdmx/3.0/structure/dataflow/"
+    "ESTAT/all/1.0?format=JSON&compress=false"
+)
+CODELISTS_URL = "https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1/codelist/ESTAT/all/latest?format=json"
+METABASE_URL = "https://ec.europa.eu/eurostat/api/dissemination/catalogue/metabase.txt.gz"
+_GLOBAL_CACHE: dict[
+    str,
+    tuple[
+        dict[str, _Dataflow],
+        dict[str, Any],
+        dict[str, dict[str, list[str]]],
+        dict[str, list[list[dict[str, str]]]],
+    ],
+] = {}
+_GLOBAL_CACHE_LOCK = asyncio.Lock()
+_CLEAN_TAGS_PATTERN = re.compile(r"<.*?>")
+
+
+def _remove_html_tags(text: str) -> str:
+    return re.sub(_CLEAN_TAGS_PATTERN, "", text)
+
+
+def _parse_toc_paths(toc_text: str) -> dict[str, list[list[dict[str, str]]]]:
+    reader = csv.reader(io.StringIO(toc_text), delimiter="\t", quotechar='"')
+    next(reader, None)
+    folder_stack: list[dict[str, str]] = []
+    full_stack: list[dict[str, str]] = []
+    code_to_paths: dict[str, list[list[dict[str, str]]]] = {}
+    for row in reader:
+        if len(row) < 3:
+            continue
+        title, code, item_type = row[0], row[1], row[2]
+        if not title.strip() or not code.strip():
+            continue
+        indent_level = (len(title) - len(title.lstrip())) // 4
+        full_stack = full_stack[:indent_level]
+        full_stack.append({"id": code, "label": title.strip(), "type": item_type})
+        folder_stack = [
+            {"id": node["id"], "label": node["label"]}
+            for node in full_stack
+            if node.get("type") == "folder"
+        ]
+        if item_type != "folder":
+            code_to_paths.setdefault(code.upper(), []).append(list(folder_stack))
+    return code_to_paths
+
+
+def _parse_metabase(gzipped_content: bytes) -> dict[str, dict[str, list[str]]]:
+    metabase: dict[str, dict[str, list[str]]] = {}
+    with gzip.open(io.BytesIO(gzipped_content), "rt", encoding="utf-8") as handle:
+        dataset_dim_values: defaultdict[str, defaultdict[str, list[str]]] = defaultdict(
+            lambda: defaultdict(list)
+        )
+        for line in handle:
+            stripped = line.strip()
+            if not stripped:
+                continue
+            parts = stripped.split("\t")
+            if len(parts) < 3:
+                continue
+            dataset_code, dimension, value = parts[:3]
+            dataset_dim_values[dataset_code.upper()][dimension].append(value)
+    for dataset_code, dimensions in dataset_dim_values.items():
+        metabase[dataset_code] = dict(dimensions)
+    return metabase
+
+
+@dataclass(slots=True)
+class _Dataflow:
+    dataset_code: str
+    label: str
+    updated: str
+    source: str | None
+    description: str | None
+    first_period: str | None
+    last_period: str | None
+    extension: dict[str, Any]
+
+
+class EurostatClient(APIWrapper):
+    def __init__(self, provider: Provider, **kwargs: Any) -> None:
+        super().__init__(
+            provider_code=provider.provider_code,
+            label=provider.label,
+            language=kwargs.pop("language"),
+            json_request_handler=kwargs.pop("json_request_handler"),
+            text_request_handler=kwargs.pop("text_request_handler", None),
+            bytes_request_handler=kwargs.pop("bytes_request_handler", None),
+            logger=kwargs.pop("logger", None),
+        )
+        self.provider = provider
+        self.api_type = provider.api_type
+
+    async def health_check(self, dataset_code: str | None = None) -> bool:
+        url = (
+            DATAFLOWS_URL
+            if dataset_code is None
+            else METADATA_URL_TEMPLATE.format(
+                dataset_code=dataset_code,
+                language=self.language,
+            )
+        )
+        payload = await self._get_json(url)
+        return payload is not None
+
+    async def _fetch_global_metadata(
+        self,
+    ) -> tuple[
+        dict[str, _Dataflow],
+        dict[str, Any],
+        dict[str, dict[str, list[str]]],
+        dict[str, list[list[dict[str, str]]]],
+    ]:
+        cache_key = self.language.lower()
+        cached = _GLOBAL_CACHE.get(cache_key)
+        if cached is not None:
+            return cached
+        async with _GLOBAL_CACHE_LOCK:
+            cached = _GLOBAL_CACHE.get(cache_key)
+            if cached is not None:
+                return cached
+            toc_text, dataflows_payload, codelists_payload, metabase_payload = (
+                await asyncio.gather(
+                    self._get_text(TOC_URL_TEMPLATE.format(language=self.language)),
+                    self._get_json(DATAFLOWS_URL),
+                    self._get_json(CODELISTS_URL),
+                    self._get_bytes(METABASE_URL),
+                )
+            )
+            dataflows: dict[str, _Dataflow] = {}
+            if isinstance(dataflows_payload, dict):
+                items = dataflows_payload.get("link", {}).get("item", [])
+                for item in items:
+                    if not isinstance(item, dict):
+                        continue
+                    extension = item.get("extension", {})
+                    if not isinstance(extension, dict):
+                        extension = {}
+                    dataset_code = str(extension.get("id") or "").upper()
+                    label = str(item.get("label") or "").strip()
+                    if not dataset_code or not label:
+                        continue
+                    annotation_map: dict[str, Any] = {}
+                    for annotation in extension.get("annotation", []):
+                        if not isinstance(annotation, dict):
+                            continue
+                        annotation_type = annotation.get("type")
+                        if not annotation_type:
+                            continue
+                        for key in ("date", "title", "text"):
+                            if key in annotation:
+                                annotation_map[str(annotation_type)] = annotation[key]
+                                break
+                    updated = max(
+                        str(annotation_map.get("UPDATE_DATA") or ""),
+                        str(annotation_map.get("UPDATE_STRUCTURE") or ""),
+                    )
+                    if not updated:
+                        continue
+                    description = extension.get("description")
+                    dataflows[dataset_code] = _Dataflow(
+                        dataset_code=dataset_code,
+                        label=label,
+                        updated=updated,
+                        source=str(annotation_map.get("SOURCE_INSTITUTIONS") or "Eurostat"),
+                        description=_remove_html_tags(description)
+                        if isinstance(description, str)
+                        else None,
+                        first_period=str(
+                            annotation_map.get("OBS_PERIOD_OVERALL_OLDEST") or ""
+                        ).strip()
+                        or None,
+                        last_period=str(
+                            annotation_map.get("OBS_PERIOD_OVERALL_LATEST") or ""
+                        ).strip()
+                        or None,
+                        extension={"dataflow_extension": extension} if extension else {},
+                    )
+            codelists = codelists_payload if isinstance(codelists_payload, dict) else {}
+            metabase = (
+                _parse_metabase(metabase_payload)
+                if isinstance(metabase_payload, bytes)
+                else {}
+            )
+            toc_paths = _parse_toc_paths(toc_text) if isinstance(toc_text, str) else {}
+            _GLOBAL_CACHE[cache_key] = (dataflows, codelists, metabase, toc_paths)
+            return _GLOBAL_CACHE[cache_key]
+
+    def _build_dimensions(
+        self,
+        dataset_code: str,
+        codelists: dict[str, Any],
+        metabase: dict[str, dict[str, list[str]]],
+    ) -> tuple[dict[str, dict[str, Any]], list[str], dict[str, list[str]], str]:
+        dimensions: dict[str, dict[str, Any]] = {}
+        role: dict[str, list[str]] = {}
+        time_unit = "Other"
+        items = codelists.get("link", {}).get("item", []) if isinstance(codelists, dict) else []
+        for dimension_id, category_ids in metabase.get(dataset_code, {}).items():
+            if dimension_id.lower() == "time":
+                time_unit = (
+                    "Annual"
+                    if category_ids and all(len(value) == 4 for value in category_ids[:2])
+                    else "Other"
+                )
+                labels = {value: value for value in category_ids}
+                index = {value: ordinal for ordinal, value in enumerate(category_ids)}
+                dimensions[dimension_id] = {
+                    "label": "Time",
+                    "category": {"index": index, "label": labels},
+                }
+                role.setdefault("time", []).append(dimension_id)
+                continue
+
+            dimension_label = dimension_id
+            label_map = {value: value for value in category_ids}
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                extension = item.get("extension", {})
+                if not isinstance(extension, dict):
+                    continue
+                if str(extension.get("id") or "").upper() != dimension_id.upper():
+                    continue
+                dimension_label = str(item.get("label") or dimension_id)
+                labels = item.get("category", {}).get("label", {})
+                if isinstance(labels, dict):
+                    label_map = {
+                        value: str(labels.get(value, value)) for value in category_ids
+                    }
+                break
+
+            dimensions[dimension_id] = {
+                "label": dimension_label,
+                "category": {
+                    "index": {
+                        value: ordinal for ordinal, value in enumerate(category_ids)
+                    },
+                    "label": label_map,
+                },
+            }
+            if dimension_id.lower() == "geo":
+                role.setdefault("geo", []).append(dimension_id)
+        return dimensions, list(dimensions.keys()), role, time_unit
+
+    async def discover_datasets(
+        self,
+        task_id: uuid.UUID,
+        **_: Any,
+    ) -> list[DiscoveredDataset]:
+        dataflows, codelists, metabase, toc_paths = await self._fetch_global_metadata()
+        output: list[DiscoveredDataset] = []
+        for dataset_code, dataflow in dataflows.items():
+            if "$dv_" in dataset_code.lower():
+                continue
+            dimensions, dimension_ids, role, time_unit = self._build_dimensions(
+                dataset_code,
+                codelists,
+                metabase,
+            )
+            updated = parse_dt(dataflow.updated)
+            if updated is None:
+                continue
+            paths = toc_paths.get(dataset_code, [])
+            subject = paths[0][0] if paths and paths[0] else None
+            output.append(
+                DiscoveredDataset(
+                    task_id=task_id,
+                    provider_code=self.provider_code,
+                    dataset_code=dataset_code,
+                    language=self.language,
+                    updated=updated,
+                    label=_remove_html_tags(dataflow.label),
+                    source=dataflow.source,
+                    description=dataflow.description,
+                    time_unit=time_unit,
+                    first_period=dataflow.first_period,
+                    last_period=dataflow.last_period,
+                    paths=paths or None,
+                    subject_code=subject["id"] if subject else None,
+                    subject_label=subject["label"] if subject else None,
+                    metadata_url=METADATA_URL_TEMPLATE.format(
+                        dataset_code=dataset_code,
+                        language=self.language,
+                    ),
+                    data_url=DATA_URL_TEMPLATE.format(dataset_code=dataset_code),
+                    web_url=WEB_URL_TEMPLATE.format(dataset_code=dataset_code),
+                    doc_url=DOC_URL_TEMPLATE.format(
+                        lowercase_dataset_code=dataset_code.lower()
+                    ),
+                    dimension=dimensions,
+                    required_dimensions=dict.fromkeys(dimension_ids, None),
+                    role=role,
+                    dimension_ids=dimension_ids,
+                    extension=dataflow.extension,
+                )
+            )
+        return output
+
+    async def resolve_dataset_metadata(
+        self,
+        discovered: DiscoveredDataset,
+        task_id: uuid.UUID | None = None,
+        **_: Any,
+    ) -> ResolvedDatasetMetadata:
+        return ResolvedDatasetMetadata(
+            task_id=task_id or discovered.task_id,
+            provider_code=discovered.provider_code,
+            dataset_code=discovered.dataset_code,
+            language=discovered.language,
+            updated=discovered.updated,
+            label=discovered.label or discovered.dataset_code,
+            time_unit=discovered.time_unit or "Other",
+            first_period=discovered.first_period or "",
+            last_period=discovered.last_period or discovered.first_period or "",
+            paths=discovered.paths or [],
+            role=discovered.role or {},
+            metadata_url=discovered.metadata_url or "",
+            data_url=discovered.data_url or "",
+            dimension=discovered.dimension or {},
+            required_dimensions=discovered.required_dimensions or {},
+            note=discovered.note,
+            source=discovered.source or "Eurostat",
+            description=discovered.description,
+            discontinued=discovered.discontinued,
+            subject_code=discovered.subject_code,
+            subject_label=discovered.subject_label,
+            web_url=discovered.web_url,
+            doc_url=discovered.doc_url,
+            official_statistics=discovered.official_statistics,
+            contact=discovered.contact,
+            dimension_ids=discovered.dimension_ids,
+            extension=discovered.extension,
+        )