sdmxlib 0.8.0__py3-none-any.whl

sdmxlib/formats/sdmx_ml21/reader.py

@@ -0,0 +1,1147 @@
+"""SDMX-ML 2.1 reader: lxml → domain model.
+
+Collapses thirteen XML wrapper layers into clean domain objects.
+All references within the message are auto-resolved.
+"""
+
+from datetime import datetime
+from typing import Any
+
+from lxml import etree
+
+from sdmxlib.model.annotations import Annotation, Annotations
+from sdmxlib.model.category import Categorisation, Category, CategoryScheme
+from sdmxlib.model.codelist import Code, Codelist
+from sdmxlib.model.collections import ArtefactList, ItemList
+from sdmxlib.model.concept import Concept, ConceptScheme
+from sdmxlib.model.constraint import (
+    ConstraintAttachment,
+    ContentConstraint,
+    CubeRegion,
+    KeyValue,
+    ReferencePeriod,
+    ReleaseCalendar,
+)
+from sdmxlib.model.dataflow import Dataflow
+from sdmxlib.model.datastructure import (
+    AttachmentLevel,
+    Attribute,
+    DataStructure,
+    Dimension,
+    Group,
+    Measure,
+    MeasureDimension,
+    TimeDimension,
+)
+from sdmxlib.model.hierarchy import HierarchicalCode, Hierarchy
+from sdmxlib.model.istring import InternationalString
+from sdmxlib.model.mapping import (
+    CategoryMap,
+    CategorySchemeMap,
+    CodelistMap,
+    CodeMap,
+    ComponentMap,
+    ConceptMap,
+    ConceptSchemeMap,
+    DataflowMap,
+    DataStructureMap,
+    StructureSet,
+)
+from sdmxlib.model.message import StructureMessage
+from sdmxlib.model.organisation import Agency, AgencyScheme, DataProvider, DataProviderScheme
+from sdmxlib.model.provision import ProvisionAgreement
+from sdmxlib.model.ref import AnyRef, Ref
+from sdmxlib.model.registry import Registry
+from sdmxlib.model.representation import Facet
+from sdmxlib.model.urn import SdmxUrn
+
+from .namespaces import COM, MES, STR, XML_LANG
+
+# ── Clark-notation tag builders ───────────────────────────────────────────────
+
+
+def _c(tag: str) -> str:
+    return f"{{{COM}}}{tag}"
+
+
+def _s(tag: str) -> str:
+    return f"{{{STR}}}{tag}"
+
+
+def _m(tag: str) -> str:
+    return f"{{{MES}}}{tag}"
+
+
+# ── Common XML helpers ────────────────────────────────────────────────────────
+
+
+def _read_istring(el: etree._Element, child_tag: str = "Name") -> InternationalString:
+    """Read com:Name (or com:Description) children as InternationalString."""
+    langs: dict[str, str] = {}
+    for child in el.findall(_c(child_tag)):
+        lang = child.get(XML_LANG, "")
+        if lang and child.text:
+            langs[lang] = child.text
+    return InternationalString(**langs)
+
+
+def _read_annotations(el: etree._Element) -> Annotations:
+    """Read com:Annotations block."""
+    anns_el = el.find(_c("Annotations"))
+    if anns_el is None:
+        return Annotations()
+    items: list[Annotation] = []
+    for ann_el in anns_el.findall(_c("Annotation")):
+        title_el = ann_el.find(_c("AnnotationTitle"))
+        type_el = ann_el.find(_c("AnnotationType"))
+        url_el = ann_el.find(_c("AnnotationURL"))
+        text_langs: dict[str, str] = {}
+        for text_el in ann_el.findall(_c("AnnotationText")):
+            lang = text_el.get(XML_LANG, "")
+            if lang and text_el.text:
+                text_langs[lang] = text_el.text
+        items.append(
+            Annotation(
+                id=ann_el.get("id"),
+                title=title_el.text if title_el is not None else None,
+                type=type_el.text if type_el is not None else None,
+                url=url_el.text if url_el is not None else None,
+                text=InternationalString(**text_langs),
+            )
+        )
+    return Annotations(items)
+
+
+def _read_datetime_attr(el: etree._Element, attr: str) -> datetime | None:
+    """Parse an optional xs:dateTime attribute (e.g. validFrom, validTo)."""
+    val = el.get(attr)
+    if not val:
+        return None
+    return datetime.fromisoformat(val)
+
+
+# ── Agency lookup ─────────────────────────────────────────────────────────────
+
+
+def _resolve_agency(agency_id: str, agencies: dict[str, Agency]) -> Agency:
+    """Return the rich Agency if parsed from an AgencyScheme, else a minimal one."""
+    return agencies.get(agency_id, Agency(id=agency_id))
+
+
+# ── Ref / URN helpers ─────────────────────────────────────────────────────────
+
+
+def _ref_el_to_urn(ref_el: etree._Element) -> SdmxUrn[Any] | None:
+    """Convert a Ref element to a SdmxUrn.
+
+    Handles both URN-attribute refs and component-attribute refs, including
+    item refs that carry maintainableParentID.
+    """
+    if urn_str := ref_el.get("urn"):
+        return SdmxUrn.parse(urn_str)
+
+    agency = ref_el.get("agencyID", "")
+    id_ = ref_el.get("id", "")
+    version = ref_el.get("version", "latest")
+    package = ref_el.get("package", "")
+    cls = ref_el.get("class", "")
+
+    if not (agency and id_ and package and cls):
+        return None
+
+    parent_id = ref_el.get("maintainableParentID")
+    parent_version = ref_el.get("maintainableParentVersion", "latest")
+
+    if parent_id:
+        # Item URN: id_ is the item, parent_id is the maintainable container
+        return SdmxUrn.make(
+            object,
+            package=package,
+            artefact_class=cls,
+            agency=agency,
+            id=parent_id,
+            version=parent_version,
+            item_id=id_,
+        )
+    return SdmxUrn.make(
+        object,
+        package=package,
+        artefact_class=cls,
+        agency=agency,
+        id=id_,
+        version=version,
+    )
+
+
+def _resolve_ref(container_el: etree._Element, index: dict[str, Any]) -> AnyRef | None:
+    """Find a direct Ref child and resolve it against the index."""
+    ref_el = container_el.find("Ref")
+    if ref_el is None:
+        return None
+    urn = _ref_el_to_urn(ref_el)
+    if urn is None:
+        return None
+    obj = index.get(str(urn))
+    if obj is not None:
+        return Ref.of(urn, obj)
+    return Ref.unresolved(urn)
+
+
+# ── Index helpers ─────────────────────────────────────────────────────────────
+
+
+def _index_artefact(index: dict[str, Any], obj: Any, artefact_type: type[Any]) -> None:
+    """Add a maintainable artefact to the resolution index under both versioned and latest keys."""
+    package = artefact_type.sdmx_package
+    cls = artefact_type.sdmx_class
+    urn = SdmxUrn.make(
+        artefact_type,
+        package=package,
+        artefact_class=cls,
+        agency=obj.agency_id,
+        id=obj.id,
+        version=obj.version,
+    )
+    index[str(urn)] = obj
+    urn_latest = SdmxUrn.make(artefact_type, package=package, artefact_class=cls, agency=obj.agency_id, id=obj.id)
+    index[str(urn_latest)] = obj
+
+
+def _index_item(
+    index: dict[str, Any],
+    item: Any,
+    parent: Any,
+    package: str,
+    cls: str,
+) -> None:
+    """Add an item (e.g. Concept) to the index under its item URN."""
+    urn = SdmxUrn.make(
+        object,
+        package=package,
+        artefact_class=cls,
+        agency=parent.agency_id,
+        id=parent.id,
+        version=parent.version,
+        item_id=item.id,
+    )
+    index[str(urn)] = item
+    urn_latest = SdmxUrn.make(
+        object,
+        package=package,
+        artefact_class=cls,
+        agency=parent.agency_id,
+        id=parent.id,
+        item_id=item.id,
+    )
+    index[str(urn_latest)] = item
+
+
+# ── Representation ────────────────────────────────────────────────────────────
+
+
+def _read_text_format(tf_el: etree._Element) -> Facet:
+    def _int(v: str | None) -> int | None:
+        return int(v) if v is not None else None
+
+    def _float(v: str | None) -> float | None:
+        return float(v) if v is not None else None
+
+    def _bool(v: str | None) -> bool | None:
+        if v is None:
+            return None
+        return v.lower() in ("true", "1")
+
+    return Facet(
+        type=tf_el.get("textType"),
+        max_length=_int(tf_el.get("maxLength")),
+        min_length=_int(tf_el.get("minLength")),
+        max_value=_float(tf_el.get("maxValue")),
+        min_value=_float(tf_el.get("minValue")),
+        decimals=_int(tf_el.get("decimals")),
+        pattern=tf_el.get("pattern"),
+        start_value=_float(tf_el.get("startValue")),
+        end_value=_float(tf_el.get("endValue")),
+        is_sequence=_bool(tf_el.get("isSequence")),
+        is_multi_lingual=_bool(tf_el.get("isMultiLingual")),
+    )
+
+
+def _read_representation(local_rep_el: etree._Element, index: dict[str, Any] | None = None) -> AnyRef | Facet | None:
+    """Parse a LocalRepresentation or CoreRepresentation element.
+
+    When *index* is provided the codelist ref is resolved against it.
+    Without *index* (concept CoreRepresentation parsing) an unresolved Ref is returned.
+    """
+    enum_el = local_rep_el.find(_s("Enumeration"))
+    if enum_el is not None:
+        if index is not None:
+            return _resolve_ref(enum_el, index)
+        ref_el = enum_el.find("Ref")
+        if ref_el is not None:
+            urn = _ref_el_to_urn(ref_el)
+            if urn is not None:
+                return Ref.unresolved(urn)
+
+    tf_el = local_rep_el.find(_s("TextFormat"))
+    if tf_el is not None:
+        return _read_text_format(tf_el)
+
+    return None
+
+
+# ── OrganisationSchemes ───────────────────────────────────────────────────────
+
+
+def _read_organisation_schemes(
+    structures: etree._Element,
+) -> tuple[list[AgencyScheme], list[DataProviderScheme]]:
+    container = structures.find(_s("OrganisationSchemes"))
+    if container is None:
+        return [], []
+
+    agency_schemes: list[AgencyScheme] = []
+    provider_schemes: list[DataProviderScheme] = []
+
+    for as_el in container.findall(_s("AgencyScheme")):
+        agencies = [
+            Agency(
+                id=a.get("id", ""),
+                name=_read_istring(a),
+                description=_read_istring(a, "Description"),
+                annotations=_read_annotations(a),
+            )
+            for a in as_el.findall(_s("Agency"))
+        ]
+        scheme = AgencyScheme(
+            id=as_el.get("id", ""),
+            maintainer=as_el.get("agencyID", ""),
+            version=as_el.get("version", "1.0"),
+            name=_read_istring(as_el),
+            description=_read_istring(as_el, "Description"),
+            is_final=as_el.get("isFinal", "false").lower() == "true",
+            agencies=ItemList(agencies),
+            annotations=_read_annotations(as_el),
+            valid_from=_read_datetime_attr(as_el, "validFrom"),
+            valid_to=_read_datetime_attr(as_el, "validTo"),
+        )
+        agency_schemes.append(scheme)
+
+    for ps_el in container.findall(_s("DataProviderScheme")):
+        providers = [
+            DataProvider(
+                id=p.get("id", ""),
+                name=_read_istring(p),
+                description=_read_istring(p, "Description"),
+                annotations=_read_annotations(p),
+            )
+            for p in ps_el.findall(_s("DataProvider"))
+        ]
+        scheme_p = DataProviderScheme(
+            id=ps_el.get("id", ""),
+            maintainer=ps_el.get("agencyID", ""),
+            version=ps_el.get("version", "1.0"),
+            name=_read_istring(ps_el),
+            description=_read_istring(ps_el, "Description"),
+            is_final=ps_el.get("isFinal", "false").lower() == "true",
+            providers=ItemList(providers),
+            annotations=_read_annotations(ps_el),
+            valid_from=_read_datetime_attr(ps_el, "validFrom"),
+            valid_to=_read_datetime_attr(ps_el, "validTo"),
+        )
+        provider_schemes.append(scheme_p)
+
+    return agency_schemes, provider_schemes
+
+
+# ── Codelists ─────────────────────────────────────────────────────────────────
+
+
+def _read_codelists(structures: etree._Element, agencies: dict[str, Agency]) -> list[Codelist]:
+    container = structures.find(_s("Codelists"))
+    if container is None:
+        return []
+    result: list[Codelist] = []
+    for cl_el in container.findall(_s("Codelist")):
+        codes: list[Code] = []
+        for code_el in cl_el.findall(_s("Code")):
+            parent_id: str | None = None
+            parent_el = code_el.find(_s("Parent"))
+            if parent_el is not None:
+                ref_el = parent_el.find("Ref")
+                if ref_el is not None:
+                    parent_id = ref_el.get("id")
+            codes.append(
+                Code(
+                    id=code_el.get("id", ""),
+                    name=_read_istring(code_el),
+                    description=_read_istring(code_el, "Description"),
+                    parent_id=parent_id,
+                    annotations=_read_annotations(code_el),
+                )
+            )
+        result.append(
+            Codelist(
+                id=cl_el.get("id", ""),
+                maintainer=_resolve_agency(cl_el.get("agencyID", ""), agencies),
+                version=cl_el.get("version", "1.0"),
+                name=_read_istring(cl_el),
+                description=_read_istring(cl_el, "Description"),
+                is_final=cl_el.get("isFinal", "false").lower() == "true",
+                codes=ItemList(codes),
+                annotations=_read_annotations(cl_el),
+                valid_from=_read_datetime_attr(cl_el, "validFrom"),
+                valid_to=_read_datetime_attr(cl_el, "validTo"),
+            )
+        )
+    return result
+
+
+# ── ConceptSchemes ────────────────────────────────────────────────────────────
+
+
+def _read_concept_schemes(structures: etree._Element, agencies: dict[str, Agency]) -> list[ConceptScheme]:
+    container = structures.find(_s("Concepts"))
+    if container is None:
+        return []
+    result: list[ConceptScheme] = []
+    for cs_el in container.findall(_s("ConceptScheme")):
+        concepts = []
+        for concept_el in cs_el.findall(_s("Concept")):
+            core_rep = None
+            core_rep_el = concept_el.find(_s("CoreRepresentation"))
+            if core_rep_el is not None:
+                core_rep = _read_representation(core_rep_el)
+            concepts.append(
+                Concept(
+                    id=concept_el.get("id", ""),
+                    name=_read_istring(concept_el),
+                    description=_read_istring(concept_el, "Description"),
+                    core_representation=core_rep,
+                    annotations=_read_annotations(concept_el),
+                )
+            )
+        result.append(
+            ConceptScheme(
+                id=cs_el.get("id", ""),
+                maintainer=_resolve_agency(cs_el.get("agencyID", ""), agencies),
+                version=cs_el.get("version", "1.0"),
+                name=_read_istring(cs_el),
+                description=_read_istring(cs_el, "Description"),
+                concepts=ItemList(concepts),
+                annotations=_read_annotations(cs_el),
+                valid_from=_read_datetime_attr(cs_el, "validFrom"),
+                valid_to=_read_datetime_attr(cs_el, "validTo"),
+            )
+        )
+    return result
+
+
+# ── DataStructures ────────────────────────────────────────────────────────────
+
+
+def _read_dimension(dim_el: etree._Element, index: dict[str, Any]) -> Dimension:
+    concept_ref: AnyRef | None = None
+    ci_el = dim_el.find(_s("ConceptIdentity"))
+    if ci_el is not None:
+        concept_ref = _resolve_ref(ci_el, index)
+
+    representation = None
+    local_rep_el = dim_el.find(_s("LocalRepresentation"))
+    if local_rep_el is not None:
+        representation = _read_representation(local_rep_el, index)
+
+    return Dimension(
+        id=dim_el.get("id", ""),
+        concept=concept_ref,
+        representation=representation,
+        annotations=_read_annotations(dim_el),
+    )
+
+
+def _read_measure_dimension(md_el: etree._Element, index: dict[str, Any]) -> MeasureDimension:
+    concept_ref: AnyRef | None = None
+    ci_el = md_el.find(_s("ConceptIdentity"))
+    if ci_el is not None:
+        concept_ref = _resolve_ref(ci_el, index)
+
+    representation = None
+    local_rep_el = md_el.find(_s("LocalRepresentation"))
+    if local_rep_el is not None:
+        representation = _read_representation(local_rep_el, index)
+
+    return MeasureDimension(
+        id=md_el.get("id", ""),
+        concept=concept_ref,
+        representation=representation,
+        annotations=_read_annotations(md_el),
+    )
+
+
+def _read_time_dimension(td_el: etree._Element, index: dict[str, Any]) -> TimeDimension:
+    concept_ref: AnyRef | None = None
+    ci_el = td_el.find(_s("ConceptIdentity"))
+    if ci_el is not None:
+        concept_ref = _resolve_ref(ci_el, index)
+
+    facet_rep: Facet | None = None
+    local_rep_el = td_el.find(_s("LocalRepresentation"))
+    if local_rep_el is not None:
+        tf_el = local_rep_el.find(_s("TextFormat"))
+        if tf_el is not None:
+            facet_rep = _read_text_format(tf_el)
+
+    return TimeDimension(
+        id=td_el.get("id", "TIME_PERIOD"),
+        concept=concept_ref,
+        representation=facet_rep,
+        annotations=_read_annotations(td_el),
+    )
+
+
+def _read_attachment(
+    ar_el: etree._Element,
+) -> tuple[AttachmentLevel | None, list[str] | None, str | None]:
+    """Parse AttributeRelationship → (level, related_dimensions, related_measure)."""
+    pm_el = ar_el.find(_s("PrimaryMeasure"))
+    if pm_el is not None:
+        ref_el = pm_el.find("Ref")
+        measure_id = ref_el.get("id") if ref_el is not None else "OBS_VALUE"
+        return AttachmentLevel.OBSERVATION, None, measure_id
+
+    dim_refs = ar_el.findall(_s("Dimension"))
+    if dim_refs:
+        related = [ref_el.get("id", "") for d in dim_refs if (ref_el := d.find("Ref")) is not None]
+        return AttachmentLevel.SERIES, related or None, None
+
+    if ar_el.find(_s("Group")) is not None:
+        return AttachmentLevel.GROUP, None, None
+
+    return AttachmentLevel.DATASET, None, None
+
+
+def _read_attribute(attr_el: etree._Element, index: dict[str, Any]) -> Attribute:
+    concept_ref: AnyRef | None = None
+    ci_el = attr_el.find(_s("ConceptIdentity"))
+    if ci_el is not None:
+        concept_ref = _resolve_ref(ci_el, index)
+
+    representation = None
+    local_rep_el = attr_el.find(_s("LocalRepresentation"))
+    if local_rep_el is not None:
+        representation = _read_representation(local_rep_el, index)
+
+    attachment: AttachmentLevel | None = None
+    related_dims: list[str] | None = None
+    related_measure: str | None = None
+    ar_el = attr_el.find(_s("AttributeRelationship"))
+    if ar_el is not None:
+        attachment, related_dims, related_measure = _read_attachment(ar_el)
+
+    return Attribute(
+        id=attr_el.get("id", ""),
+        usage=attr_el.get("assignmentStatus"),
+        attachment=attachment,
+        related_dimensions=related_dims,
+        related_measure=related_measure,
+        concept=concept_ref,
+        representation=representation,
+        annotations=_read_annotations(attr_el),
+    )
+
+
+def _read_measure(pm_el: etree._Element, index: dict[str, Any]) -> Measure:
+    concept_ref: AnyRef | None = None
+    ci_el = pm_el.find(_s("ConceptIdentity"))
+    if ci_el is not None:
+        concept_ref = _resolve_ref(ci_el, index)
+
+    representation = None
+    local_rep_el = pm_el.find(_s("LocalRepresentation"))
+    if local_rep_el is not None:
+        representation = _read_representation(local_rep_el, index)
+
+    return Measure(
+        id=pm_el.get("id", "OBS_VALUE"),
+        concept=concept_ref,
+        representation=representation,
+        annotations=_read_annotations(pm_el),
+    )
+
+
+def _read_groups(components_el: etree._Element) -> list[Group]:
+    result: list[Group] = []
+    for group_el in components_el.findall(_s("Group")):
+        dim_ids: list[str] = []
+        for gd_el in group_el.findall(_s("GroupDimension")):
+            dr_el = gd_el.find(_s("DimensionReference"))
+            if dr_el is not None:
+                ref_el = dr_el.find("Ref")
+                if ref_el is not None:
+                    dim_ids.append(ref_el.get("id", ""))
+        result.append(Group(id=group_el.get("id", ""), dimension_ids=dim_ids))
+    return result
+
+
+def _read_dsd_components(
+    components_el: etree._Element, index: dict[str, Any]
+) -> tuple[list[Dimension], TimeDimension | None, MeasureDimension | None, list[Attribute], list[Measure], list[Group]]:
+    """Parse DataStructureComponents into component lists."""
+    dimensions: list[Dimension] = []
+    time_dimension: TimeDimension | None = None
+    measure_dimension: MeasureDimension | None = None
+    attributes: list[Attribute] = []
+    measures: list[Measure] = []
+
+    dim_list_el = components_el.find(_s("DimensionList"))
+    if dim_list_el is not None:
+        dimensions.extend(_read_dimension(d, index) for d in dim_list_el.findall(_s("Dimension")))
+        td_el = dim_list_el.find(_s("TimeDimension"))
+        if td_el is not None:
+            time_dimension = _read_time_dimension(td_el, index)
+        md_el = dim_list_el.find(_s("MeasureDimension"))
+        if md_el is not None:
+            measure_dimension = _read_measure_dimension(md_el, index)
+
+    attr_list_el = components_el.find(_s("AttributeList"))
+    if attr_list_el is not None:
+        attributes.extend(_read_attribute(a, index) for a in attr_list_el.findall(_s("Attribute")))
+
+    measure_list_el = components_el.find(_s("MeasureList"))
+    if measure_list_el is not None:
+        pm_el = measure_list_el.find(_s("PrimaryMeasure"))
+        if pm_el is not None:
+            measures.append(_read_measure(pm_el, index))
+
+    groups = _read_groups(components_el)
+    return dimensions, time_dimension, measure_dimension, attributes, measures, groups
+
+
+def _read_data_structures(
+    structures: etree._Element,
+    index: dict[str, Any],
+    agencies: dict[str, Agency],
+) -> list[DataStructure]:
+    container = structures.find(_s("DataStructures"))
+    if container is None:
+        return []
+    result: list[DataStructure] = []
+    for dsd_el in container.findall(_s("DataStructure")):
+        dimensions: list[Dimension] = []
+        time_dimension: TimeDimension | None = None
+        measure_dimension: MeasureDimension | None = None
+        attributes: list[Attribute] = []
+        measures: list[Measure] = []
+        groups: list[Group] = []
+
+        components_el = dsd_el.find(_s("DataStructureComponents"))
+        if components_el is not None:
+            dimensions, time_dimension, measure_dimension, attributes, measures, groups = _read_dsd_components(
+                components_el, index
+            )
+
+        result.append(
+            DataStructure(
+                id=dsd_el.get("id", ""),
+                maintainer=_resolve_agency(dsd_el.get("agencyID", ""), agencies),
+                version=dsd_el.get("version", "1.0"),
+                name=_read_istring(dsd_el),
+                description=_read_istring(dsd_el, "Description"),
+                is_final=dsd_el.get("isFinal", "false").lower() == "true",
+                dimensions=ItemList(dimensions),
+                time_dimension=time_dimension,
+                measure_dimension=measure_dimension,
+                attributes=ItemList(attributes),
+                measures=ItemList(measures),
+                groups=ItemList(groups) if groups else None,
+                annotations=_read_annotations(dsd_el),
+                valid_from=_read_datetime_attr(dsd_el, "validFrom"),
+                valid_to=_read_datetime_attr(dsd_el, "validTo"),
+            )
+        )
+    return result
+
+
+# ── Dataflows ─────────────────────────────────────────────────────────────────
+
+
+def _read_dataflows(
+    structures: etree._Element,
+    index: dict[str, Any],
+    agencies: dict[str, Agency],
+) -> list[Dataflow]:
+    container = structures.find(_s("Dataflows"))
+    if container is None:
+        return []
+    result: list[Dataflow] = []
+    for df_el in container.findall(_s("Dataflow")):
+        structure_ref: AnyRef | None = None
+        struct_el = df_el.find(_s("Structure"))
+        if struct_el is not None:
+            structure_ref = _resolve_ref(struct_el, index)
+        result.append(
+            Dataflow(
+                id=df_el.get("id", ""),
+                maintainer=_resolve_agency(df_el.get("agencyID", ""), agencies),
+                version=df_el.get("version", "1.0"),
+                name=_read_istring(df_el),
+                description=_read_istring(df_el, "Description"),
+                annotations=_read_annotations(df_el),
+                structure=structure_ref,
+                valid_from=_read_datetime_attr(df_el, "validFrom"),
+                valid_to=_read_datetime_attr(df_el, "validTo"),
+            )
+        )
+    return result
+
+
+# ── CategorySchemes & Categorisations ─────────────────────────────────────────
+
+
+def _read_categories(parent_el: etree._Element) -> list[Category]:
+    """Recursively read Category elements."""
+    return [
+        Category(
+            id=cat_el.get("id", ""),
+            name=_read_istring(cat_el),
+            description=_read_istring(cat_el, "Description"),
+            categories=ItemList(_read_categories(cat_el)),
+            annotations=_read_annotations(cat_el),
+        )
+        for cat_el in parent_el.findall(_s("Category"))
+    ]
+
+
+def _read_category_schemes(structures: etree._Element, agencies: dict[str, Agency]) -> list[CategoryScheme]:
+    container = structures.find(_s("CategorySchemes"))
+    if container is None:
+        return []
+    return [
+        CategoryScheme(
+            id=cs_el.get("id", ""),
+            maintainer=_resolve_agency(cs_el.get("agencyID", ""), agencies),
+            version=cs_el.get("version", "1.0"),
+            name=_read_istring(cs_el),
+            description=_read_istring(cs_el, "Description"),
+            categories=ItemList(_read_categories(cs_el)),
+            annotations=_read_annotations(cs_el),
+            valid_from=_read_datetime_attr(cs_el, "validFrom"),
+            valid_to=_read_datetime_attr(cs_el, "validTo"),
+        )
+        for cs_el in container.findall(_s("CategoryScheme"))
+    ]
+
+
+def _read_categorisations(
+    structures: etree._Element,
+    index: dict[str, Any],
+    agencies: dict[str, Agency],
+) -> list[Categorisation]:
+    container = structures.find(_s("Categorisations"))
+    if container is None:
+        return []
+    result: list[Categorisation] = []
+    for cat_el in container.findall(_s("Categorisation")):
+        source_ref: AnyRef | None = None
+        source_el = cat_el.find(_s("Source"))
+        if source_el is not None:
+            source_ref = _resolve_ref(source_el, index)
+
+        target_ref: AnyRef | None = None
+        target_el = cat_el.find(_s("Target"))
+        if target_el is not None:
+            target_ref = _resolve_ref(target_el, index)
+
+        result.append(
+            Categorisation(
+                id=cat_el.get("id", ""),
+                maintainer=_resolve_agency(cat_el.get("agencyID", ""), agencies),
+                version=cat_el.get("version", "1.0"),
+                name=_read_istring(cat_el),
+                description=_read_istring(cat_el, "Description"),
+                annotations=_read_annotations(cat_el),
+                source=source_ref,
+                target=target_ref,
+                valid_from=_read_datetime_attr(cat_el, "validFrom"),
+                valid_to=_read_datetime_attr(cat_el, "validTo"),
+            )
+        )
+    return result
+
+
+# ── ProvisionAgreements ───────────────────────────────────────────────────────
+
+
+def _read_provision_agreements(
+    structures: etree._Element,
+    index: dict[str, Any],
+    agencies: dict[str, Agency],
+) -> list[ProvisionAgreement]:
+    container = structures.find(_s("ProvisionAgreements"))
+    if container is None:
+        return []
+    result: list[ProvisionAgreement] = []
+    for pa_el in container.findall(_s("ProvisionAgreement")):
+        dataflow_ref: AnyRef | None = None
+        su_el = pa_el.find(_s("StructureUsage"))
+        if su_el is not None:
+            dataflow_ref = _resolve_ref(su_el, index)
+
+        provider_ref: AnyRef | None = None
+        dp_el = pa_el.find(_s("DataProvider"))
+        if dp_el is not None:
+            provider_ref = _resolve_ref(dp_el, index)
+
+        result.append(
+            ProvisionAgreement(
+                id=pa_el.get("id", ""),
+                maintainer=_resolve_agency(pa_el.get("agencyID", ""), agencies),
+                version=pa_el.get("version", "1.0"),
+                name=_read_istring(pa_el),
+                description=_read_istring(pa_el, "Description"),
+                annotations=_read_annotations(pa_el),
+                dataflow=dataflow_ref,
+                data_provider=provider_ref,
+                valid_from=_read_datetime_attr(pa_el, "validFrom"),
+                valid_to=_read_datetime_attr(pa_el, "validTo"),
+            )
+        )
+    return result
+
+
+# ── Constraints ───────────────────────────────────────────────────────────────
+
+
+def _read_reference_period(cc_el: etree._Element) -> ReferencePeriod | None:
+    rp_el = cc_el.find(_s("ReferencePeriod"))
+    if rp_el is None:
+        return None
+    return ReferencePeriod(
+        start_period=datetime.fromisoformat(rp_el.get("startTime", "")),
+        end_period=datetime.fromisoformat(rp_el.get("endTime", "")),
+    )
+
+
+def _read_release_calendar(cc_el: etree._Element) -> ReleaseCalendar | None:
+    rc_el = cc_el.find(_s("ReleaseCalendar"))
+    if rc_el is None:
+        return None
+    periodicity_el = rc_el.find(_s("Periodicity"))
+    offset_el = rc_el.find(_s("Offset"))
+    tolerance_el = rc_el.find(_s("Tolerance"))
+    return ReleaseCalendar(
+        periodicity=periodicity_el.text if periodicity_el is not None and periodicity_el.text else "",
+        offset=offset_el.text if offset_el is not None and offset_el.text else "",
+        tolerance=tolerance_el.text if tolerance_el is not None and tolerance_el.text else "",
+    )
+
+
+def _read_constraints(
+    structures: etree._Element,
+    index: dict[str, Any],
+    agencies: dict[str, Agency],
+) -> list[ContentConstraint]:
+    container = structures.find(_s("Constraints"))
+    if container is None:
+        return []
+    result: list[ContentConstraint] = []
+    for cc_el in container.findall(_s("ContentConstraint")):
+        # ConstraintAttachment
+        attach = ConstraintAttachment()
+        att_el = cc_el.find(_s("ConstraintAttachment"))
+        if att_el is not None:
+            df_el = att_el.find(_s("Dataflow"))
+            if df_el is not None:
+                ref = _resolve_ref(df_el, index)
+                attach = ConstraintAttachment(dataflow=ref)
+            else:
+                ds_el = att_el.find(_s("DataStructure"))
+                if ds_el is not None:
+                    ref = _resolve_ref(ds_el, index)
+                    attach = ConstraintAttachment(data_structure=ref)
+                else:
+                    pa_el = att_el.find(_s("ProvisionAgreement"))
+                    if pa_el is not None:
+                        ref = _resolve_ref(pa_el, index)
+                        attach = ConstraintAttachment(provision_agreement=ref)
+
+        # CubeRegions
+        regions: list[CubeRegion] = []
+        for region_el in cc_el.findall(_s("CubeRegion")):
+            include_str = region_el.get("include", "true")
+            include = include_str.lower() != "false"
+            keys: list[KeyValue] = []
+            for kv_el in region_el.findall(_c("KeyValue")):
+                dim_id = kv_el.get("id", "")
+                values = frozenset(v.text for v in kv_el.findall(_c("Value")) if v.text)
+                keys.append(KeyValue(dimension_id=dim_id, values=values))
+            regions.append(CubeRegion(include=include, keys=tuple(keys)))
+
+        result.append(
+            ContentConstraint(
+                id=cc_el.get("id", ""),
+                maintainer=_resolve_agency(cc_el.get("agencyID", ""), agencies),
+                version=cc_el.get("version", "1.0"),
+                name=_read_istring(cc_el),
+                description=_read_istring(cc_el, "Description"),
+                is_final=cc_el.get("isFinal", "false").lower() == "true",
+                constraint_type=cc_el.get("type", "Allowed"),
+                attachment=attach,
+                regions=tuple(regions),
+                reference_period=_read_reference_period(cc_el),
+                release_calendar=_read_release_calendar(cc_el),
+                annotations=_read_annotations(cc_el),
+                valid_from=_read_datetime_attr(cc_el, "validFrom"),
+                valid_to=_read_datetime_attr(cc_el, "validTo"),
+            )
+        )
+    return result
+
+
+# ── StructureSets ─────────────────────────────────────────────────────────────
+
+
+def _read_source_target_refs(el: etree._Element, index: dict[str, Any]) -> tuple[AnyRef | None, AnyRef | None]:
+    """Read Source and Target ref children from a map element."""
+    source_ref = None
+    src_el = el.find(_s("Source"))
+    if src_el is not None:
+        source_ref = _resolve_ref(src_el, index)
+    target_ref = None
+    tgt_el = el.find(_s("Target"))
+    if tgt_el is not None:
+        target_ref = _resolve_ref(tgt_el, index)
+    return source_ref, target_ref
+
+
+def _read_id_pair_maps(parent: etree._Element, tag: str, cls: type[Any]) -> list[Any]:
+    """Read item maps that have Source/Target children with Ref id attributes."""
+    results: list[Any] = []
+    for m_el in parent.findall(_s(tag)):
+        m_src = m_el.find(_s("Source"))
+        m_tgt = m_el.find(_s("Target"))
+        if m_src is not None and m_tgt is not None:
+            src_ref_el = m_src.find("Ref")
+            tgt_ref_el = m_tgt.find("Ref")
+            if src_ref_el is not None and tgt_ref_el is not None:
+                results.append(
+                    cls(
+                        source_id=src_ref_el.get("id", ""),
+                        target_id=tgt_ref_el.get("id", ""),
+                    )
+                )
+    return results
+
+
+def _read_scheme_map(
+    parent: etree._Element,
+    scheme_tag: str,
+    scheme_cls: type[Any],
+    item_tag: str,
+    item_cls: type[Any],
+    index: dict[str, Any],
+) -> list[Any]:
+    """Read a list of scheme maps (CodelistMap, ConceptSchemeMap, etc.)."""
+    results: list[Any] = []
+    for el in parent.findall(_s(scheme_tag)):
+        source_ref, target_ref = _read_source_target_refs(el, index)
+        item_maps = _read_id_pair_maps(el, item_tag, item_cls)
+        results.append(
+            scheme_cls(
+                id=el.get("id", ""),
+                name=_read_istring(el),
+                description=_read_istring(el, "Description"),
+                source=source_ref,
+                target=target_ref,
+                maps=tuple(item_maps),
+            )
+        )
+    return results
+
+
+def _read_structure_sets(
+    structures: etree._Element,
+    index: dict[str, Any],
+    agencies: dict[str, Agency],
+) -> list[StructureSet]:
+    container = structures.find(_s("StructureSets"))
+    if container is None:
+        return []
+    result: list[StructureSet] = []
+    for ss_el in container.findall(_s("StructureSet")):
+        # DataflowMaps (no item maps)
+        dataflow_maps: list[DataflowMap] = []
+        for dfm_el in ss_el.findall(_s("DataflowMap")):
+            source_ref, target_ref = _read_source_target_refs(dfm_el, index)
+            dataflow_maps.append(
+                DataflowMap(
+                    id=dfm_el.get("id", ""),
+                    name=_read_istring(dfm_el),
+                    description=_read_istring(dfm_el, "Description"),
+                    source=source_ref,
+                    target=target_ref,
+                )
+            )
+
+        result.append(
+            StructureSet(
+                id=ss_el.get("id", ""),
+                maintainer=_resolve_agency(ss_el.get("agencyID", ""), agencies),
+                version=ss_el.get("version", "1.0"),
+                name=_read_istring(ss_el),
+                description=_read_istring(ss_el, "Description"),
+                is_final=ss_el.get("isFinal", "false").lower() == "true",
+                codelist_maps=tuple(_read_scheme_map(ss_el, "CodelistMap", CodelistMap, "CodeMap", CodeMap, index)),
+                concept_scheme_maps=tuple(
+                    _read_scheme_map(ss_el, "ConceptSchemeMap", ConceptSchemeMap, "ConceptMap", ConceptMap, index)
+                ),
+                category_scheme_maps=tuple(
+                    _read_scheme_map(ss_el, "CategorySchemeMap", CategorySchemeMap, "CategoryMap", CategoryMap, index)
+                ),
+                data_structure_maps=tuple(
+                    _read_scheme_map(ss_el, "DataStructureMap", DataStructureMap, "ComponentMap", ComponentMap, index)
+                ),
+                dataflow_maps=tuple(dataflow_maps),
+                annotations=_read_annotations(ss_el),
+                valid_from=_read_datetime_attr(ss_el, "validFrom"),
+                valid_to=_read_datetime_attr(ss_el, "validTo"),
+            )
+        )
+    return result
+
+
+# ── Hierarchies ───────────────────────────────────────────────────────────────
+
+
+def _read_hierarchical_code(hc_el: etree._Element) -> HierarchicalCode:
+    """Recursively parse a HierarchicalCode element."""
+    code_ref: SdmxUrn[Any] | None = None
+    code_id_el = hc_el.find(_s("CodeID"))
+    if code_id_el is not None:
+        ref_el = code_id_el.find("Ref")
+        if ref_el is not None:
+            agency = ref_el.get("agencyID", "")
+            parent_id = ref_el.get("maintainableParentID", "")
+            item_id = ref_el.get("id", "")
+            version = ref_el.get("version", "latest")
+            if agency and parent_id and item_id:
+                code_ref = SdmxUrn.make(
+                    object,
+                    package="codelist",
+                    artefact_class="Code",
+                    agency=agency,
+                    id=parent_id,
+                    version=version,
+                    item_id=item_id,
+                )
+
+    children = [_read_hierarchical_code(child_el) for child_el in hc_el.findall(_s("HierarchicalCode"))]
+
+    return HierarchicalCode(
+        id=hc_el.get("id", ""),
+        code=code_ref,
+        name=_read_istring(hc_el),
+        description=_read_istring(hc_el, "Description"),
+        children=children,
+    )
+
+
+def _read_hierarchies(
+    structures: etree._Element,
+    agencies: dict[str, Agency],
+) -> list[Hierarchy]:
+    container = structures.find(_s("HierarchicalCodelists"))
+    if container is None:
+        return []
+    result: list[Hierarchy] = []
+    for hcl_el in container.findall(_s("HierarchicalCodelist")):
+        hcl_agency = hcl_el.get("agencyID", "")
+        hcl_version = hcl_el.get("version", "1.0")
+        for h_el in hcl_el.findall(_s("Hierarchy")):
+            tree = [_read_hierarchical_code(hc_el) for hc_el in h_el.findall(_s("HierarchicalCode"))]
+            result.append(
+                Hierarchy(
+                    id=h_el.get("id", ""),
+                    maintainer=_resolve_agency(hcl_agency, agencies),
+                    version=hcl_version,
+                    name=_read_istring(h_el),
+                    description=_read_istring(h_el, "Description"),
+                    annotations=_read_annotations(h_el),
+                    tree=tree,
+                )
+            )
+    return result
+
+
+# ── Entry point ───────────────────────────────────────────────────────────────
+
+
+def read(data: bytes, *, registry: Registry | None = None) -> StructureMessage:
+    """Parse SDMX-ML 2.1 bytes into a StructureMessage.
+
+    References within the message are auto-resolved.
+
+    If *registry* is provided all parsed artefacts are added to it via
+    ``registry.add_all()``, interning their Refs for cross-message resolution.
+    """
+    root = etree.fromstring(data)
+    structures = root.find(_m("Structures"))
+    if structures is None:
+        return StructureMessage()
+
+    # ── Phase 1: parse leaf artefacts (no refs to other artefacts) ──────────
+    agency_schemes, data_provider_schemes = _read_organisation_schemes(structures)
+
+    # Build agency lookup for opportunistic resolution in subsequent phases
+    agencies: dict[str, Agency] = {a.id: a for s in agency_schemes for a in s.agencies}
+
+    codelists = _read_codelists(structures, agencies)
+    concept_schemes = _read_concept_schemes(structures, agencies)
+    category_schemes = _read_category_schemes(structures, agencies)
+
+    # ── Phase 2: build resolution index ─────────────────────────────────────
+    index: dict[str, Any] = {}
+
+    for cl in codelists:
+        _index_artefact(index, cl, Codelist)
+
+    for cs in concept_schemes:
+        _index_artefact(index, cs, ConceptScheme)
+        for concept in cs.concepts:
+            _index_item(index, concept, cs, ConceptScheme.sdmx_package, "Concept")
+
+    for cat_s in category_schemes:
+        _index_artefact(index, cat_s, CategoryScheme)
+
+    # ── Phase 3: parse artefacts that reference codelists / concepts ─────────
+    data_structures = _read_data_structures(structures, index, agencies)
+    for dsd in data_structures:
+        _index_artefact(index, dsd, DataStructure)
+
+    dataflows = _read_dataflows(structures, index, agencies)
+    for df in dataflows:
+        _index_artefact(index, df, Dataflow)
+
+    # ── Phase 4: parse artefacts that reference DSDs / dataflows ────────────
+    categorisations = _read_categorisations(structures, index, agencies)
+    provision_agreements = _read_provision_agreements(structures, index, agencies)
+    constraints = _read_constraints(structures, index, agencies)
+    structure_sets = _read_structure_sets(structures, index, agencies)
+    hierarchies = _read_hierarchies(structures, agencies)
+
+    msg = StructureMessage(
+        codelists=ArtefactList(codelists),
+        concept_schemes=ArtefactList(concept_schemes),
+        data_structures=ArtefactList(data_structures),
+        dataflows=ArtefactList(dataflows),
+        category_schemes=ArtefactList(category_schemes),
+        categorisations=ArtefactList(categorisations),
+        provision_agreements=ArtefactList(provision_agreements),
+        agency_schemes=ArtefactList(agency_schemes),
+        data_provider_schemes=ArtefactList(data_provider_schemes),
+        constraints=ArtefactList(constraints),
+        structure_sets=ArtefactList(structure_sets),
+        hierarchies=ArtefactList(hierarchies),
+    )
+    if registry is not None:
+        registry.add_all(msg.all_artefacts())
+    return msg