dv_schema_models 0.9.0__tar.gz

dv_schema_models-0.9.0/PKG-INFO

@@ -0,0 +1,216 @@
+Metadata-Version: 2.4
+Name: dv_schema_models
+Version: 0.9.0
+Summary: Turns Harvard Dataverse Project metadatablocks schema and dataset JSON into Pydantic models.
+Author: Ken Lui
+Author-email: Ken Lui <kenlh.lui@utoronto.ca>
+License-Expression: MIT
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Python: >=3.10, <4.0
+Project-URL: homepage, https://github.com/kenlhlui/dv_schema_models
+Project-URL: source, https://github.com/kenlhlui/dv_schema_models
+Project-URL: changelog, https://github.com/kenlhlui/dv_schema_models/blob/main/CHANGELOG.md
+Project-URL: releasenotes, https://github.com/kenlhlui/dv_schema_models/releases
+Project-URL: documentation, https://kenlhlui.github.io/dv_schema_models
+Project-URL: issues, https://github.com/kenlhlui/dv_schema_models/issues
+Description-Content-Type: text/markdown
+
+# dv_schema_models
+
+Pydantic models for Dataverse metadata — parse the schema, load dataset 
+exports, and validate field values against the schema.
+
+> [!CAUTION]
+> This library is under active development and the API is not yet stable. Breaking changes may occur between releases. Please pin to a specific version in your `pyproject.toml` or `requirements.txt` if you want to avoid surprises.
+
+## Pre-requisites
+1. Python 3.10+
+
+## Installation
+
+1. With `uv` (recommended):
+```bash
+uv add dv_schema_models
+```
+
+2. With `pip`:
+```bash
+pip install dv_schema_models
+```
+
+To export schemas to Excel (see [usage #5](#5-export-the-schema-to-a-spreadsheet)), install with the `spreadsheet` extra:
+```bash
+uv add "dv_schema_models[spreadsheet]"   # or: pip install "dv_schema_models[spreadsheet]"
+```
+
+## Concepts
+
+| Thing | What it is |
+|---|---|
+| **Schema** | `/api/metadatablocks` response — defines what fields *can* exist, their types, and rules |
+| **Dataset instance** | `GET /api/datasets/:id` response — the actual metadata values for one dataset |
+| **Record model** | A Pydantic model *generated from* the schema, used to validate instance values |
+
+## Usage
+
+### 1. Load and query the schema
+
+```python
+import json
+from dv_schema_models.dataverse_schema import load_schema
+
+schema = load_schema(json.load(open("dv_schema.json")))
+
+schema.block_names()                        # ['citation', 'geospatial', ...]
+block = schema.get_block("citation")
+block.fields.keys()                         # top-level field names
+block.required_fields()                     # leaf fields where isRequired=True
+block.all_leaf_fields()                     # flattened, including nested compound fields
+
+field = block.get_field("keyword")
+field.is_compound()                         # True — has childFields
+field.iter_leaf_fields()                    # [keywordValue, keywordVocabulary, ...]
+```
+
+### 2. Load a dataset and read values
+
+```python
+import json
+from dv_schema_models.dataset_instance import IsPartOf, load_dataset
+
+dataset = load_dataset(json.load(open("ds_metadata.json")))
+
+# Load the possible typeNames for a given block
+dataset.field_names("citation")  # ['title', 'author', 'keyword', ...] 
+dataset.data.latestVersion.metadataBlocks.get("citation").field_names() # same
+
+
+# Shortcut from the top level
+dataset.get_value("citation", "title")      # plain string
+
+# Or drill down
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+block.get_value("keyword")                  # unwrapped Python value (str / list / dict)
+block.get_field("author").simple_value()    # [{'authorName': 'Author1', 'authorAffiliation': 'Author1Aff'...} ... {'authorName': 'Author2', 'authorAffiliation': 'Author2Aff'...}]
+
+# Pull one subfield out of a compound field
+block.get_subfield_values("author", "authorName")  # ['Author1', 'Author2']
+
+# Walk the isPartOf chain (dataset -> collection -> parent collection -> ...), possibly None
+IsPartOf.get_field_list(dataset.data.isPartOf, "identifier")  # ['sub-collection', 'top-collection']
+```
+
+### 3. Work with files
+
+```python
+from dv_schema_models.dataset_instance import load_dataset
+from dv_schema_models.file_instance import FileInstance
+
+dataset = load_dataset(json.load(open("ds_metadata.json")))
+
+files = dataset.data.latestVersion.files or []
+FileInstance.sum_field(files, "filesize")   # sum a DataFile field across files, e.g. total filesize
+FileInstance.list_field(files, "dataFile.filename")   # list a field's values across files, dotted path for nested fields
+```
+
+`sum_field` skips files with no `dataFile` or a `None` value for the field. Returns `None` (and logs a warning) if any present value isn't numeric.
+
+`list_field` takes a dotted path (e.g. `"restricted"` for a top-level field, `"dataFile.checksum.type"` for a nested one) and skips entries where the path is missing or `None`.
+
+### 4. Work with role assignments
+
+```python
+import json
+from dv_schema_models.role_assignments import load_role_assignments
+
+role_assignments = load_role_assignments(json.load(open("ds_role_assignments.json")))
+
+role_assignments.count_field("assignee")            # number of assignments with an "assignee" field
+role_assignments.count_field("roleName", "Curator") # number of assignments where roleName == "Curator"
+role_assignments.get_value("assignee")              # ['@personA', '@personB', ...]
+
+# Fields not on the schema (e.g. the `_roleAlias` Dataverse sends) are still reachable
+role_assignments.data[0].get_raw("_roleAlias")      # 'curator'
+```
+
+`load_role_assignments` also accepts the error envelope Dataverse returns when the request isn't permitted (`{"status": "ERROR", "message": "..."}`) — `data` is `None`, and `message` is reachable via `role_assignments.model_extra`.
+
+### 5. Validate instance values against the schema
+
+```python
+import json
+from dv_schema_models.dataverse_schema import load_schema
+from dv_schema_models.dataset_instance import load_dataset
+from dv_schema_models.schema_driven_records import build_record_model, flatten_instance
+
+
+schema = load_schema(json.load(open("dv_schema.json")))
+dataset = load_dataset(json.load(open("ds_metadata.json")))
+
+citation_schema = schema.get_block("citation")
+CitationRecord = build_record_model(citation_schema)   # dynamic Pydantic model
+
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+raw = flatten_instance(block)              # {typeName: value, ...}
+record = CitationRecord.model_validate(raw)
+```
+
+The generated model enforces field names, required/optional status, list wrapping for `multiple=True` fields, and `int`/`float` types where declared by the schema.
+
+### 6. Discover available fields
+
+```python
+# Fields actually present in this dataset instance
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+block.field_names()                            # e.g. ['title', 'author', 'keyword', ...]
+
+# All fields the schema defines (including absent/optional ones)
+schema.get_block("citation").all_leaf_fields().keys()
+
+# After validation, access as typed attributes
+record = CitationRecord.model_validate(flatten_instance(block))
+record.title          # str
+record.author         # list[...] for multiple=True compound fields
+record.keyword        # None if not present in this dataset (optional fields default to None)
+# Note: field names with dots become underscores — e.g. 'resolution.Spatial' → record.resolution_Spatial
+```
+
+### 7. Export the schema to a spreadsheet
+
+Requires the `spreadsheet` extra (see [Installation](#installation)).
+
+```python
+import json
+from dv_schema_models.dataverse_schema import load_schema
+from dv_schema_models.schema_spreadsheet import SchemaSpreadsheet
+
+schema = load_schema(json.load(open("dv_schema.json")))
+SchemaSpreadsheet(schema).write("dv_schema.xlsx")
+```
+
+Writes an `.xlsx` workbook with one formatted worksheet per metadata block plus a combined **All** sheet. See [docs/schema_spreadsheet/README.md](docs/schema_spreadsheet/README.md) for the output layout, column mapping, and architecture.
+
+## Input file shapes
+
+**Schema** — output of Dataverse `/api/metadatablocks`:
+```json
+{"status": "OK", "data": [{"id": 10, "name": "citation", "fields": {...}}]}
+```
+
+**Dataset** — output of Dataverse `GET /api/datasets/:id`:
+```json
+{"status": "OK", "data": {"latestVersion": {"metadataBlocks": {"citation": {"fields": [...]}}}}}
+```
+
+**Role assignments** — output of Dataverse `GET /api/datasets/:id/assignments`:
+```json
+{"status": "OK", "data": [{"id": 1, "assignee": "@user", "roleId": 7, "roleName": "Curator", "definitionPointId": 34847}]}
+```
+Error responses (e.g. `{"status": "ERROR", "message": "..."}`, no `data` key) are also accepted — see [usage #4](#4-work-with-role-assignments).
+
+## Citation
+If you use this library in your work, please cite according to [CITATION](CITATION.cff)
+
+## License
+[MIT](LICENSE)

dv_schema_models-0.9.0/README.md

@@ -0,0 +1,198 @@
+# dv_schema_models
+
+Pydantic models for Dataverse metadata — parse the schema, load dataset 
+exports, and validate field values against the schema.
+
+> [!CAUTION]
+> This library is under active development and the API is not yet stable. Breaking changes may occur between releases. Please pin to a specific version in your `pyproject.toml` or `requirements.txt` if you want to avoid surprises.
+
+## Pre-requisites
+1. Python 3.10+
+
+## Installation
+
+1. With `uv` (recommended):
+```bash
+uv add dv_schema_models
+```
+
+2. With `pip`:
+```bash
+pip install dv_schema_models
+```
+
+To export schemas to Excel (see [usage #5](#5-export-the-schema-to-a-spreadsheet)), install with the `spreadsheet` extra:
+```bash
+uv add "dv_schema_models[spreadsheet]"   # or: pip install "dv_schema_models[spreadsheet]"
+```
+
+## Concepts
+
+| Thing | What it is |
+|---|---|
+| **Schema** | `/api/metadatablocks` response — defines what fields *can* exist, their types, and rules |
+| **Dataset instance** | `GET /api/datasets/:id` response — the actual metadata values for one dataset |
+| **Record model** | A Pydantic model *generated from* the schema, used to validate instance values |
+
+## Usage
+
+### 1. Load and query the schema
+
+```python
+import json
+from dv_schema_models.dataverse_schema import load_schema
+
+schema = load_schema(json.load(open("dv_schema.json")))
+
+schema.block_names()                        # ['citation', 'geospatial', ...]
+block = schema.get_block("citation")
+block.fields.keys()                         # top-level field names
+block.required_fields()                     # leaf fields where isRequired=True
+block.all_leaf_fields()                     # flattened, including nested compound fields
+
+field = block.get_field("keyword")
+field.is_compound()                         # True — has childFields
+field.iter_leaf_fields()                    # [keywordValue, keywordVocabulary, ...]
+```
+
+### 2. Load a dataset and read values
+
+```python
+import json
+from dv_schema_models.dataset_instance import IsPartOf, load_dataset
+
+dataset = load_dataset(json.load(open("ds_metadata.json")))
+
+# Load the possible typeNames for a given block
+dataset.field_names("citation")  # ['title', 'author', 'keyword', ...] 
+dataset.data.latestVersion.metadataBlocks.get("citation").field_names() # same
+
+
+# Shortcut from the top level
+dataset.get_value("citation", "title")      # plain string
+
+# Or drill down
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+block.get_value("keyword")                  # unwrapped Python value (str / list / dict)
+block.get_field("author").simple_value()    # [{'authorName': 'Author1', 'authorAffiliation': 'Author1Aff'...} ... {'authorName': 'Author2', 'authorAffiliation': 'Author2Aff'...}]
+
+# Pull one subfield out of a compound field
+block.get_subfield_values("author", "authorName")  # ['Author1', 'Author2']
+
+# Walk the isPartOf chain (dataset -> collection -> parent collection -> ...), possibly None
+IsPartOf.get_field_list(dataset.data.isPartOf, "identifier")  # ['sub-collection', 'top-collection']
+```
+
+### 3. Work with files
+
+```python
+from dv_schema_models.dataset_instance import load_dataset
+from dv_schema_models.file_instance import FileInstance
+
+dataset = load_dataset(json.load(open("ds_metadata.json")))
+
+files = dataset.data.latestVersion.files or []
+FileInstance.sum_field(files, "filesize")   # sum a DataFile field across files, e.g. total filesize
+FileInstance.list_field(files, "dataFile.filename")   # list a field's values across files, dotted path for nested fields
+```
+
+`sum_field` skips files with no `dataFile` or a `None` value for the field. Returns `None` (and logs a warning) if any present value isn't numeric.
+
+`list_field` takes a dotted path (e.g. `"restricted"` for a top-level field, `"dataFile.checksum.type"` for a nested one) and skips entries where the path is missing or `None`.
+
+### 4. Work with role assignments
+
+```python
+import json
+from dv_schema_models.role_assignments import load_role_assignments
+
+role_assignments = load_role_assignments(json.load(open("ds_role_assignments.json")))
+
+role_assignments.count_field("assignee")            # number of assignments with an "assignee" field
+role_assignments.count_field("roleName", "Curator") # number of assignments where roleName == "Curator"
+role_assignments.get_value("assignee")              # ['@personA', '@personB', ...]
+
+# Fields not on the schema (e.g. the `_roleAlias` Dataverse sends) are still reachable
+role_assignments.data[0].get_raw("_roleAlias")      # 'curator'
+```
+
+`load_role_assignments` also accepts the error envelope Dataverse returns when the request isn't permitted (`{"status": "ERROR", "message": "..."}`) — `data` is `None`, and `message` is reachable via `role_assignments.model_extra`.
+
+### 5. Validate instance values against the schema
+
+```python
+import json
+from dv_schema_models.dataverse_schema import load_schema
+from dv_schema_models.dataset_instance import load_dataset
+from dv_schema_models.schema_driven_records import build_record_model, flatten_instance
+
+
+schema = load_schema(json.load(open("dv_schema.json")))
+dataset = load_dataset(json.load(open("ds_metadata.json")))
+
+citation_schema = schema.get_block("citation")
+CitationRecord = build_record_model(citation_schema)   # dynamic Pydantic model
+
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+raw = flatten_instance(block)              # {typeName: value, ...}
+record = CitationRecord.model_validate(raw)
+```
+
+The generated model enforces field names, required/optional status, list wrapping for `multiple=True` fields, and `int`/`float` types where declared by the schema.
+
+### 6. Discover available fields
+
+```python
+# Fields actually present in this dataset instance
+block = dataset.data.latestVersion.metadataBlocks.get("citation")
+block.field_names()                            # e.g. ['title', 'author', 'keyword', ...]
+
+# All fields the schema defines (including absent/optional ones)
+schema.get_block("citation").all_leaf_fields().keys()
+
+# After validation, access as typed attributes
+record = CitationRecord.model_validate(flatten_instance(block))
+record.title          # str
+record.author         # list[...] for multiple=True compound fields
+record.keyword        # None if not present in this dataset (optional fields default to None)
+# Note: field names with dots become underscores — e.g. 'resolution.Spatial' → record.resolution_Spatial
+```
+
+### 7. Export the schema to a spreadsheet
+
+Requires the `spreadsheet` extra (see [Installation](#installation)).
+
+```python
+import json
+from dv_schema_models.dataverse_schema import load_schema
+from dv_schema_models.schema_spreadsheet import SchemaSpreadsheet
+
+schema = load_schema(json.load(open("dv_schema.json")))
+SchemaSpreadsheet(schema).write("dv_schema.xlsx")
+```
+
+Writes an `.xlsx` workbook with one formatted worksheet per metadata block plus a combined **All** sheet. See [docs/schema_spreadsheet/README.md](docs/schema_spreadsheet/README.md) for the output layout, column mapping, and architecture.
+
+## Input file shapes
+
+**Schema** — output of Dataverse `/api/metadatablocks`:
+```json
+{"status": "OK", "data": [{"id": 10, "name": "citation", "fields": {...}}]}
+```
+
+**Dataset** — output of Dataverse `GET /api/datasets/:id`:
+```json
+{"status": "OK", "data": {"latestVersion": {"metadataBlocks": {"citation": {"fields": [...]}}}}}
+```
+
+**Role assignments** — output of Dataverse `GET /api/datasets/:id/assignments`:
+```json
+{"status": "OK", "data": [{"id": 1, "assignee": "@user", "roleId": 7, "roleName": "Curator", "definitionPointId": 34847}]}
+```
+Error responses (e.g. `{"status": "ERROR", "message": "..."}`, no `data` key) are also accepted — see [usage #4](#4-work-with-role-assignments).
+
+## Citation
+If you use this library in your work, please cite according to [CITATION](CITATION.cff)
+
+## License
+[MIT](LICENSE)

dv_schema_models-0.9.0/pyproject.toml

@@ -0,0 +1,158 @@
+[build-system]  # https://docs.astral.sh/uv/concepts/build-backend/
+requires = ["uv_build>=0.11.31,<0.12"]
+build-backend = "uv_build"
+
+[project]  # https://packaging.python.org/en/latest/specifications/pyproject-toml/
+name = "dv_schema_models"
+version = "0.9.0"
+description = "Turns Harvard Dataverse Project metadatablocks schema and dataset JSON into Pydantic models."
+readme = "README.md"
+
+authors = [
+  { name = "Ken Lui", email = "kenlh.lui@utoronto.ca" },
+]
+requires-python = ">=3.10,<4.0"
+dependencies = [
+  "pydantic (>=2.0.0)",
+  "pydantic-settings (>=2.0.0)",
+]
+license = "MIT"
+
+[project.urls]  # https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels
+homepage = "https://github.com/kenlhlui/dv_schema_models"
+source = "https://github.com/kenlhlui/dv_schema_models"
+changelog = "https://github.com/kenlhlui/dv_schema_models/blob/main/CHANGELOG.md"
+releasenotes = "https://github.com/kenlhlui/dv_schema_models/releases"
+documentation = "https://kenlhlui.github.io/dv_schema_models"
+issues = "https://github.com/kenlhlui/dv_schema_models/issues"
+
+[dependency-groups]  # https://docs.astral.sh/uv/concepts/projects/dependencies/#development-dependencies
+dev = [
+  "codespell (>=2.4.1)",
+  "commitizen (>=4.3.0)",
+  "coverage[toml] (>=7.6.10)",
+  "ipykernel (>=6.29.4)",
+  "ipython (>=8.18.0)",
+  "ipywidgets (>=8.1.2)",
+  "zensical (>=0.0.51)",
+  "mkdocstrings[python] (>=0.26.2)",
+  "poethepoet (>=0.32.1)",
+  "prek",
+  "pytest (>=8.3.4)",
+  "pytest-mock (>=3.14.0)",
+  "pytest-xdist (>=3.6.1)",
+  "ruff (>=0.9.2)",
+  "ty (>=0.0.6)",
+  "mkdocstrings-python (>=2.0.5)",
+  "httpx2>=2.5.0",
+]
+
+[tool.codespell]  # https://github.com/codespell-project/codespell
+builtin = "en-GB_to_en-US,clear,code,rare"
+check-filenames = true
+skip = "./*_cache/,./.venv,./reports"
+
+[tool.commitizen]  # https://commitizen-tools.github.io/commitizen/config/
+bump_message = "bump: v$current_version → v$new_version"
+tag_format = "v$version"
+update_changelog_on_bump = true
+version_provider = "uv"
+version_scheme = "pep440"
+
+[tool.coverage.report]  # https://coverage.readthedocs.io/en/latest/config.html#report
+precision = 1
+show_missing = true
+skip_covered = true
+
+[tool.coverage.run]  # https://coverage.readthedocs.io/en/latest/config.html#run
+branch = true
+command_line = "--module pytest"
+data_file = "reports/.coverage"
+source = ["src"]
+
+[tool.coverage.xml]  # https://coverage.readthedocs.io/en/latest/config.html#xml
+output = "reports/coverage.xml"
+
+[tool.pytest.ini_options]  # https://docs.pytest.org/en/latest/reference/reference.html#ini-options-ref
+addopts = "--color=yes --doctest-modules --exitfirst --failed-first --verbosity=2 --junitxml=reports/pytest.xml"
+testpaths = ["src", "tests"]
+xfail_strict = true
+
+[tool.ruff]  # https://docs.astral.sh/ruff/settings/
+fix = true
+line-length = 100
+src = ["src", "tests"]
+target-version = "py310"
+
+[tool.ruff.format]
+docstring-code-format = true
+skip-magic-trailing-comma = true
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = ["CPY", "FIX", "T20", "ARG001", "COM812", "D203", "D213", "E501", "PD008", "PD009", "PGH003", "RET504", "S101", "TD003"]
+unfixable = ["ERA001", "F401", "F841", "T201", "T203"]
+
+[tool.ruff.lint.flake8-annotations]
+allow-star-arg-any = true
+ignore-fully-untyped = true
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.ruff.lint.isort]
+split-on-trailing-comma = false
+
+[tool.ruff.lint.pycodestyle]
+max-doc-length = 100
+
+[tool.ruff.lint.pydocstyle]
+convention = "numpy"
+
+[tool.ty.environment]
+python-platform = "all"
+
+[tool.ty.rules]
+unused-ignore-comment = "warn"
+
+[tool.poe.executor]  # https://github.com/nat-n/poethepoet
+type = "simple"
+
+[tool.poe.tasks]
+
+  [tool.poe.tasks.docs]
+  help = "Build or serve the documentation"
+  shell = """
+    if [ $serve ]
+    then {
+      zensical serve
+    } else {
+      zensical build
+    } fi
+    """
+
+    [[tool.poe.tasks.docs.args]]
+    help = "Serve the documentation locally with live reload"
+    type = "boolean"
+    name = "serve"
+    options = ["--serve"]
+
+  [tool.poe.tasks.lint]
+  help = "Lint this package"
+  cmd = """
+    prek run
+      --all-files
+      --color always
+    """
+
+  [tool.poe.tasks.test]
+  help = "Test this package"
+
+    [[tool.poe.tasks.test.sequence]]
+    cmd = "coverage run"
+
+    [[tool.poe.tasks.test.sequence]]
+    cmd = "coverage report"
+
+    [[tool.poe.tasks.test.sequence]]
+    cmd = "coverage xml"

dv_schema_models-0.9.0/src/dv_schema_models/__init__.py

@@ -0,0 +1 @@
+"""dv_schema_models."""