dv-schema-models 0.2.0a0__py3-none-any.whl

dv_schema_models/dataset_instance.py

@@ -0,0 +1,165 @@
+"""Pydantic models for a Dataverse dataset export ('GET /api/datasets/:id' response).
+
+This is the companion to dataverse_schema.py: that file models the *schema*
+(what fields CAN exist and their rules), this file models actual *metadata
+values* attached to one dataset. The shapes are different — here each field
+is {typeName, multiple, typeClass, value}, and `value` itself is polymorphic
+depending on typeClass/multiple:
+
+  - primitive / controlledVocabulary, multiple=False -> a plain string
+  - primitive / controlledVocabulary, multiple=True   -> a list of strings
+  - compound, multiple=False                          -> a dict of nested fields
+  - compound, multiple=True                           -> a list of dicts of nested fields
+"""
+# ruff: noqa: N815
+
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field
+
+
+class DatasetFieldValue(BaseModel):
+    """One metadata field value, as it appears inside metadataBlocks.<block>.fields."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    typeName: str
+    multiple: bool
+    typeClass: str  # 'primitive', 'compound', or 'controlledVocabulary'
+    value: (
+        str
+        | list[str]
+        | dict[str, DatasetFieldValue]
+        | list[dict[str, DatasetFieldValue]]
+    )
+
+    def simple_value(self) -> Any:
+        """Recursively strip away the typeName/multiple/typeClass wrapper, returning plain Python values.
+
+        Compound fields become plain dicts (or lists of dicts); primitive and
+        controlledVocabulary fields are already plain strings or lists of strings.
+        """
+        if self.typeClass == "compound":
+            if isinstance(self.value, list):
+                return [
+                    {k: v.simple_value() for k, v in item.items()}
+                    for item in self.value
+                ]
+            if isinstance(self.value, dict):
+                return {k: v.simple_value() for k, v in self.value.items()}
+        return self.value
+
+    def get_fields(self) -> list[DatasetFieldValue]:
+        return [self.typeName for f in self.value]
+
+
+# Needed because DatasetFieldValue references itself by forward-reference string.
+DatasetFieldValue.model_rebuild()
+
+
+class MetadataBlockInstance(BaseModel):
+    """One populated metadata block (e.g. citation) within a dataset version."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    displayName: str
+    name: str
+    fields: list[DatasetFieldValue]
+
+    def field_names(self) -> list[str]:
+        """Return the typeName of every field present in this block instance."""
+        return [f.typeName for f in self.fields]
+
+    def get_field(self, type_name: str) -> DatasetFieldValue | None:
+        """Find a field in this block by its typeName (e.g. 'title', 'author')."""
+        return next((f for f in self.fields if f.typeName == type_name), None)
+
+    def get_value(self, type_name: str) -> Any:
+        """Get the plain (unwrapped) value of a field in this block, or None if absent."""
+        field = self.get_field(type_name)
+        return field.simple_value() if field else None
+
+
+class DatasetVersion(BaseModel):
+    """One version of a dataset, holding the actual metadata block values and files."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    id: int
+    datasetId: int
+    versionState: str
+    datasetPersistentId: str
+    datasetType: (
+        str | None
+    )  # backward compatibility: some datasets /older Dataverse versions don't have this field
+    storageIdentifier: str
+    internalVersionNumber: int
+    versionState: str
+    latestVersionPublishingState: str
+    deaccessionLink: str | None
+    UNF: str | None
+    lastUpdateTime: str
+    createTime: str
+    termsOfUse: str | None
+    termsOfAccess: str | None
+    dataAccessPlace: str | None
+    fileAccessRequest: bool
+
+    metadataBlocks: dict[str, MetadataBlockInstance]
+
+    def get_value(self, block_name: str, type_name: str) -> Any:
+        """Get a field's plain value by block name (e.g. 'citation') and field typeName (e.g. 'title')."""
+        block = self.metadataBlocks.get(block_name)
+        return block.get_value(type_name) if block else None
+
+
+class DatasetData(BaseModel):
+    """The 'data' payload of a dataset export response."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    id: int
+    identifier: str
+    persistentUrl: str
+    protocol: str
+    authority: str
+    separator: str
+    publisher: str
+    storageIdentifier: str
+    datasetType: (
+        str | None
+    )  # backward compatibility: some datasets /older Dataverse versions don't have this field
+
+    latestVersion: DatasetVersion = Field(
+        alias="latestVersion",
+        validation_alias=AliasChoices(
+            "datasetVersion",
+            "latestVersion",
+        ),
+    )
+
+
+class DatasetExport(BaseModel):
+    """Top-level wrapper matching the raw JSON returned by GET /api/datasets/:id."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    status: str
+    data: DatasetData
+
+    def get_value(self, block_name: str, type_name: str) -> Any:
+        """Convenience shortcut: dataset.get_value('citation', 'title') straight from the top level."""
+        return self.data.latestVersion.get_value(block_name, type_name)
+
+
+def load_dataset(metadata: dict) -> DatasetExport:
+    """Parse a dataset export JSON payload (already loaded as a dict) into a DatasetExport.
+
+    Accepts either the full `{status, data: {...}}` export envelope, or a bare
+    `data`-shaped payload (no envelope), by trying each model in turn.
+    """
+    if "data" in metadata:
+        return DatasetExport.model_validate(metadata)
+    return DatasetExport(status="OK", data=DatasetData.model_validate(metadata))

dv_schema_models/dataverse_schema.py

@@ -0,0 +1,113 @@
+"""Pydantic models for the Dataverse '/api/metadatablocks' schema response.
+
+This models the *schema definition* JSON itself (block -> fields -> optional
+recursive childFields), not an individual dataset's metadata values. Use it to
+parse, validate, and query the schema (e.g. "what fields exist in the citation
+block?", "is authorAffiliation required?", "what controlled vocab values does
+subject accept?").
+"""
+
+# ruff: noqa: N815
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict
+
+
+class MetadataField(BaseModel):
+    """A single metadata field definition.
+
+    For compound fields (typeClass == 'compound'), `childFields` holds the
+    nested sub-fields, keyed by field name, recursively using this same model.
+    Leaf fields (primitive or controlledVocabulary) have `childFields = None`.
+    """
+
+    model_config = ConfigDict(
+        extra="ignore"
+    )  # tolerate any schema fields we did not model
+
+    name: str
+    displayName: str
+    displayOnCreate: bool
+    title: str
+    type: str  # e.g. 'TEXT', 'TEXTBOX', 'DATE', 'INT', 'FLOAT', 'URL', 'EMAIL', 'NONE'
+    typeClass: str  # 'primitive', 'compound', or 'controlledVocabulary'
+    watermark: str = ""
+    description: str = ""
+    multiple: bool
+    isControlledVocabulary: bool
+    isAdvancedSearchFieldType: bool
+    displayFormat: str = ""
+    displayOrder: int
+    isRequired: bool
+    controlledVocabularyValues: list[str] | None = None
+    childFields: dict[str, MetadataField] | None = None
+
+    def is_compound(self) -> bool:
+        """Return True if this field wraps nested childFields rather than holding a value directly."""
+        return self.childFields is not None
+
+    def iter_leaf_fields(self) -> list[MetadataField]:
+        """Recursively collect every leaf (non-compound) field reachable from this field."""
+        if self.childFields:
+            leaves: list[MetadataField] = []
+            for child in self.childFields.values():
+                leaves.extend(child.iter_leaf_fields())
+            return leaves
+        return [self]
+
+
+# Needed because MetadataField references itself by forward-reference string.
+MetadataField.model_rebuild()
+
+
+class MetadataBlock(BaseModel):
+    """A single metadata block (e.g. citation, geospatial, astrophysics)."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    id: int
+    name: str
+    displayName: str
+    displayOnCreate: bool
+    fields: dict[str, MetadataField]
+
+    def get_field(self, field_name: str) -> MetadataField | None:
+        """Look up a top-level field in this block by its key name."""
+        return self.fields.get(field_name)
+
+    def all_leaf_fields(self) -> dict[str, MetadataField]:
+        """Flatten every leaf field in this block (including nested ones) into a single dict."""
+        flat: dict[str, MetadataField] = {}
+        for field in self.fields.values():
+            for leaf in field.iter_leaf_fields():
+                flat[leaf.name] = leaf
+        return flat
+
+    def required_fields(self) -> list[str]:
+        """Return the names of all leaf fields marked isRequired = True."""
+        return [
+            name for name, field in self.all_leaf_fields().items() if field.isRequired
+        ]
+
+
+class DataverseSchemaResponse(BaseModel):
+    """Top-level wrapper matching the raw JSON returned by /api/metadatablocks."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    status: str
+    data: list[MetadataBlock]
+
+    def get_block(self, block_name: str) -> MetadataBlock | None:
+        """Look up a metadata block by its short name (e.g. 'citation', 'geospatial')."""
+        return next((b for b in self.data if b.name == block_name), None)
+
+    def block_names(self) -> list[str]:
+        """List the short names of every block in the response."""
+        return [b.name for b in self.data]
+
+
+def load_schema(metadata: dict) -> DataverseSchemaResponse:
+    """Parse a metadatablocks JSON payload (already loaded as a dict) into a DataverseSchemaResponse."""
+    return DataverseSchemaResponse.model_validate(metadata)

dv_schema_models/schema_driven_records.py

@@ -0,0 +1,94 @@
+"""Build a Pydantic model FROM the Dataverse schema, then use it to validate real dataset values.
+
+Workflow:
+  1. Load the metadatablocks schema (dataverse_schema.py) -> know what fields exist, their
+     types, whether they're required/multiple, and what compound sub-fields they contain.
+  2. Dynamically build a Pydantic model that mirrors that schema exactly, using pydantic.create_model.
+  3. Load a dataset export (dataset_instance.py) and flatten it to {typeName: value}.
+  4. Validate that flat dict against the schema-derived model, so the actual metadata is checked
+     against the same field names, required-ness, and structure the schema defines -- instead of
+     a generic model that would accept any field.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict, Field, create_model
+
+from .dataset_instance import MetadataBlockInstance
+from .dataverse_schema import MetadataBlock, MetadataField
+
+# Maps a Dataverse field 'type' to a Python type. Anything not listed here (TEXT, TEXTBOX,
+# DATE, URL, EMAIL, and controlled-vocabulary text) is treated as a plain str.
+_PRIMITIVE_TYPE_MAP: dict[str, type] = {
+    "INT": int,
+    "FLOAT": float,
+}
+
+
+def _python_type_for(field: MetadataField) -> type:
+    """Map a leaf schema field's declared 'type' to a Python type."""
+    return _PRIMITIVE_TYPE_MAP.get(field.type, str)
+
+
+def _safe_identifier(name: str) -> str:
+    """Turn a Dataverse field name that may contain dots (e.g. 'resolution.Spatial') into a valid Python identifier."""
+    return name.replace(".", "_")
+
+
+def build_record_model(
+    source: MetadataBlock | MetadataField,
+    *,
+    model_name: str | None = None,
+) -> type[BaseModel]:
+    """Recursively build a Pydantic model matching a schema MetadataBlock (or a compound MetadataField).
+
+    The resulting model accepts exactly the fields the schema defines: correct nested structure
+    for compound fields, List[...] wrapping for multiple=True fields, and required vs Optional
+    matching isRequired.
+    """
+    if isinstance(source, MetadataBlock):
+        fields = source.fields
+        default_name = f"{source.name.capitalize()}Record"
+    else:
+        fields = source.childFields or {}
+        default_name = f"{source.name.capitalize()}Record"
+
+    field_definitions: dict[str, Any] = {}
+
+    for field_name, field in fields.items():
+        py_name = _safe_identifier(field_name)
+
+        if field.is_compound():
+            base_type: Any = build_record_model(
+                field, model_name=f"{field_name.capitalize()}Record"
+            )
+        else:
+            base_type = _python_type_for(field)
+
+        if field.multiple:
+            base_type = list[base_type]
+
+        if not field.isRequired:
+            base_type = Optional[base_type]
+            default = None
+        else:
+            default = ...
+
+        if py_name != field_name:
+            # Field name isn't a valid Python identifier as-is -- alias back to the original.
+            field_definitions[py_name] = (base_type, Field(default, alias=field_name))
+        else:
+            field_definitions[py_name] = (base_type, default)
+
+    return create_model(
+        model_name or default_name,
+        __config__=ConfigDict(populate_by_name=True, extra="forbid"),
+        **field_definitions,
+    )
+
+
+def flatten_instance(block_instance: MetadataBlockInstance) -> dict[str, Any]:
+    """Turn a dataset export's field list into a flat {typeName: value} dict, ready to validate."""
+    return {f.typeName: f.simple_value() for f in block_instance.fields}

dv_schema_models-0.2.0a0.dist-info/METADATA

@@ -0,0 +1,143 @@
+Metadata-Version: 2.4
+Name: dv-schema-models
+Version: 0.2.0a0
+Summary: Turns Harvard Dataverse Project metadatablocks schema and dataset JSON into Pydantic models.
+Author-email: Ken Lui <kenlh.lui@utoronto.ca>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/kenlhlui/dv_schema_models/
+Project-URL: Documentation, https://github.com/kenlhlui/dv_schema_models/blob/main/README.md
+Project-URL: Repository, https://github.com/kenlhlui/dv_schema_models.git
+Project-URL: Issues, https://github.com/kenlhlui/dv_schema_models/issues
+Project-URL: Changelog, https://github.com/kenlhlui/dv_schema_models/blob/main/CHANGELOG.md
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pydantic>=2.13.4
+Dynamic: license-file
+
+# dv_schema_models
+
+Pydantic models for Dataverse metadata — parse the schema, load dataset 
+exports, and validate field values against the schema.
+
+> [!CAUTION]
+> This library is under active development and the API is not yet stable. Breaking changes may occur between releases. Please pin to a specific version in your `pyproject.toml` or `requirements.txt` if you want to avoid surprises.
+
+## Pre-requisites
+1. Python 3.13+ 
+
+## Installation
+
+1. With `uv` (recommended):
+```bash
+uv add dv_schema_models
+```
+
+2. With `pip`:
+```bash
+pip install dv_schema_models
+```
+
+## Concepts
+
+| Thing | What it is |
+|---|---|
+| **Schema** | `/api/metadatablocks` response — defines what fields *can* exist, their types, and rules |
+| **Dataset instance** | `GET /api/datasets/:id` response — the actual metadata values for one dataset |
+| **Record model** | A Pydantic model *generated from* the schema, used to validate instance values |
+
+## Usage
+
+### 1. Load and query the schema
+
+```python
+import json
+from dv_schema_models.dataverse_schema import load_schema
+
+schema = load_schema(json.load(open("dv_schema.json")))
+
+schema.block_names()                        # ['citation', 'geospatial', ...]
+block = schema.get_block("citation")
+block.fields.keys()                         # top-level field names
+block.required_fields()                     # leaf fields where isRequired=True
+block.all_leaf_fields()                     # flattened, including nested compound fields
+
+field = block.get_field("keyword")
+field.is_compound()                         # True — has childFields
+field.iter_leaf_fields()                    # [keywordValue, keywordVocabulary, ...]
+```
+
+### 2. Load a dataset and read values
+
+```python
+import json
+from dv_schema_models.dataset_instance import load_dataset
+
+dataset = load_dataset(json.load(open("ds_metadata.json")))
+
+# Shortcut from the top level
+dataset.get_value("citation", "title")      # plain string
+
+# Or drill down
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+block.get_value("keyword")                  # unwrapped Python value (str / list / dict)
+block.get_field("author").simple_value()    # same, from the DatasetFieldValue directly
+```
+
+### 3. Validate instance values against the schema
+
+```python
+import json
+from dv_schema_models.dataverse_schema import load_schema
+from dv_schema_models.dataset_instance import load_dataset
+from dv_schema_models.schema_driven_records import build_record_model, flatten_instance
+
+
+schema = load_schema(json.load(open("dv_schema.json")))
+dataset = load_dataset(json.load(open("ds_metadata.json")))
+
+citation_schema = schema.get_block("citation")
+CitationRecord = build_record_model(citation_schema)   # dynamic Pydantic model
+
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+raw = flatten_instance(block)              # {typeName: value, ...}
+record = CitationRecord.model_validate(raw)
+```
+
+The generated model enforces field names, required/optional status, list wrapping for `multiple=True` fields, and `int`/`float` types where declared by the schema.
+
+### 4. Discover available fields
+
+```python
+# Fields actually present in this dataset instance
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+block.field_names()                            # e.g. ['title', 'author', 'keyword', ...]
+
+# All fields the schema defines (including absent/optional ones)
+schema.get_block("citation").all_leaf_fields().keys()
+
+# After validation, access as typed attributes
+record = CitationRecord.model_validate(flatten_instance(block))
+record.title          # str
+record.author         # list[...] for multiple=True compound fields
+record.keyword        # None if not present in this dataset (optional fields default to None)
+# Note: field names with dots become underscores — e.g. 'resolution.Spatial' → record.resolution_Spatial
+```
+
+## Input file shapes
+
+**Schema** — output of Dataverse `/api/metadatablocks`:
+```json
+{"status": "OK", "data": [{"id": 10, "name": "citation", "fields": {...}}]}
+```
+
+**Dataset** — output of Dataverse `GET /api/datasets/:id`:
+```json
+{"status": "OK", "data": {"latestVersion": {"metadataBlocks": {"citation": {"fields": [...]}}}}}
+```
+
+## Citation
+If you use this library in your work, please cite according to [CITATION](CITATION.cff)
+
+## License
+[MIT](LICENSE)

dv_schema_models-0.2.0a0.dist-info/RECORD

@@ -0,0 +1,9 @@
+dv_schema_models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dv_schema_models/dataset_instance.py,sha256=mRSrUYIH4yQHFO5lbU9Lg88UCsi2KXyrpN4pLNI0gVg,5707
+dv_schema_models/dataverse_schema.py,sha256=1kl7rQg5ytau_9-v1J6SqPDVDw_7KpCdKyWqPNzmytY,4039
+dv_schema_models/schema_driven_records.py,sha256=Gjs6_bNF1fdCfifxE4c6hIxvFyn8-bRmbaZk8q_t_34,3592
+dv_schema_models-0.2.0a0.dist-info/licenses/LICENSE,sha256=rcRkUIujZwWhrbeUMrxbcnUMhy0p9zJsu4j6HxsHrgI,1068
+dv_schema_models-0.2.0a0.dist-info/METADATA,sha256=ENkcnOveS31S1WBAQFQFP1IPJgmx_kujiFL9G7oizB8,5107
+dv_schema_models-0.2.0a0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+dv_schema_models-0.2.0a0.dist-info/top_level.txt,sha256=exj3nMSY6lM22wuy0hEduAu6xVl59VR0FhsFnDNGjgw,17
+dv_schema_models-0.2.0a0.dist-info/RECORD,,

dv_schema_models-0.2.0a0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dv_schema_models-0.2.0a0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Lok Hei Lui
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dv_schema_models-0.2.0a0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dv_schema_models