Gen3SchemaDev 2.4.0__tar.gz → 2.6.0__tar.gz

{gen3schemadev-2.4.0 → gen3schemadev-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Gen3SchemaDev
-Version: 2.4.0
+Version: 2.6.0
 Summary: Tool for data modelling in Gen3
 License: Apache 2.0
 License-File: LICENSE
@@ -35,6 +35,19 @@ This repository aims to provide the documentation, learning materials, and softw
 - [Guide to creating your first dictionary](docs/gen3schemadev/first_dictionary.md)
 - [Troubleshooting](docs/gen3schemadev/troubleshooting.md)
 
+### Fixing `$ref` properties in an existing dictionary
+
+JSON Schema draft-04 (which Gen3 uses) ignores any keyword sitting as a direct sibling of `$ref`, so properties written as `{description: ..., $ref: _definitions.yaml#/X}` lose their description in the data-dictionary viewer. The `fix-refs` command rewrites such properties in place, moving the `$ref` into an `allOf` list so annotations survive resolution:
+
+```bash
+gen3schemadev fix-refs -y path/to/dictionary --dry-run   # preview changes
+gen3schemadev fix-refs -y path/to/dictionary             # rewrite in place
+```
+
+The command is idempotent and never touches bare refs, refs already inside `allOf`/`anyOf`/`oneOf`, the `properties: {$ref: ...}` merge construct, or `_definitions.yaml`/`_terms.yaml`/`_settings.yaml`.
+
+Both `fix-refs` and `validate` also warn about `description: null` placeholders anywhere in the dictionary (commonly in `_definitions.yaml`): the Gen3 metaschema requires descriptions to be strings, and such placeholders fail metaschema validation once exposed through bare or `allOf`-wrapped refs. Remove the null keys to resolve the warning.
+
 
 ## Deep dive into Gen3 Data Modelling
 *Special Thanks to Marion Shadbolt for providing the [source material](https://github.com/AustralianBioCommons/umccr-dictionary/tree/main/docs/schemas)*

{gen3schemadev-2.4.0 → gen3schemadev-2.6.0}/README.md

@@ -12,6 +12,19 @@ This repository aims to provide the documentation, learning materials, and softw
 - [Guide to creating your first dictionary](docs/gen3schemadev/first_dictionary.md)
 - [Troubleshooting](docs/gen3schemadev/troubleshooting.md)
 
+### Fixing `$ref` properties in an existing dictionary
+
+JSON Schema draft-04 (which Gen3 uses) ignores any keyword sitting as a direct sibling of `$ref`, so properties written as `{description: ..., $ref: _definitions.yaml#/X}` lose their description in the data-dictionary viewer. The `fix-refs` command rewrites such properties in place, moving the `$ref` into an `allOf` list so annotations survive resolution:
+
+```bash
+gen3schemadev fix-refs -y path/to/dictionary --dry-run   # preview changes
+gen3schemadev fix-refs -y path/to/dictionary             # rewrite in place
+```
+
+The command is idempotent and never touches bare refs, refs already inside `allOf`/`anyOf`/`oneOf`, the `properties: {$ref: ...}` merge construct, or `_definitions.yaml`/`_terms.yaml`/`_settings.yaml`.
+
+Both `fix-refs` and `validate` also warn about `description: null` placeholders anywhere in the dictionary (commonly in `_definitions.yaml`): the Gen3 metaschema requires descriptions to be strings, and such placeholders fail metaschema validation once exposed through bare or `allOf`-wrapped refs. Remove the null keys to resolve the warning.
+
 
 ## Deep dive into Gen3 Data Modelling
 *Special Thanks to Marion Shadbolt for providing the [source material](https://github.com/AustralianBioCommons/umccr-dictionary/tree/main/docs/schemas)*

{gen3schemadev-2.4.0 → gen3schemadev-2.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "Gen3SchemaDev"
-version = "2.4.0"
+version = "2.6.0"
 description = "Tool for data modelling in Gen3"
 authors = [
     {name = "JoshuaHarris391",email = "harjo391@gmail.com"}

{gen3schemadev-2.4.0 → gen3schemadev-2.6.0}/src/gen3schemadev/cli.py

@@ -20,6 +20,31 @@ from gen3schemadev.validators.metaschema_validator import validate_schema_with_m
 from importlib.metadata import version
 from gen3schemadev.ddvis import visualise_with_docker
 from gen3schemadev.validators.rule_validator import RuleValidator
+from gen3schemadev.refs import (
+    WRAPPED,
+    SKIP_REASONS,
+    fix_yaml_dir,
+    find_null_descriptions,
+    scan_dir_null_descriptions,
+)
+
+
+def print_null_description_warning(hits):
+    """
+    Print a warning block listing every null-valued 'description' key found
+    in the dictionary ("file: dotted.path" strings). No-op when hits is empty.
+    """
+    if not hits:
+        return
+    print(
+        "\nWARNING: found 'description: null' placeholders. The Gen3 metaschema "
+        "requires 'description' to be a string, and null placeholders in shared "
+        "definitions fail metaschema validation once exposed through bare or "
+        "allOf-wrapped $refs. Remove the null 'description' keys (removing adds "
+        "no shared description, so nothing leaks onto referencing properties):"
+    )
+    for hit in hits:
+        print(f"  - {hit}")
 
 
 def main():
@@ -142,7 +167,28 @@ def main():
         action="store_true",
         help="Set logging level to DEBUG"
     )
-    
+
+    # Create 'fix-refs' subcommand
+    fixrefs_parser = subparsers.add_parser(
+        "fix-refs",
+        help="Rewrite properties with '$ref' siblings into draft-04-safe allOf form (in place)"
+    )
+    fixrefs_parser.add_argument(
+        "-y", "--yamldir",
+        required=True,
+        help="Directory of Gen3 YAML files to rewrite in place (recursive)"
+    )
+    fixrefs_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Report what would change without writing files"
+    )
+    fixrefs_parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Set logging level to DEBUG"
+    )
+
     args = parser.parse_args()
     
     # Handle case where no command is provided
@@ -220,6 +266,15 @@ def main():
         elif args.yamls:
             schema_dict = bundle_yamls(args.yamls)
 
+        # Pre-resolution diagnostic: report every null 'description' up front,
+        # because the metaschema stage fails on the first resolved node schema,
+        # far away from the definition that carries the null.
+        null_hits = []
+        for schema_name, schema in schema_dict.items():
+            for hit in find_null_descriptions(schema):
+                null_hits.append(f"{schema_name}: {hit}")
+        print_null_description_warning(null_hits)
+
         for schema_name, schema in schema_dict.items():
 
             if '.' in schema_name:
@@ -306,5 +361,45 @@ def main():
         output_path = args.output or "input_example.yaml"
         write_yaml(init_yaml, output_path)
 
+    elif args.command == "fix-refs":
+        if not os.path.isdir(args.yamldir):
+            print(f"ERROR: Directory not found: {args.yamldir}", file=sys.stderr)
+            sys.exit(1)
+
+        if args.dry_run:
+            print(f"Dry run: scanning {args.yamldir} (no files will be written)")
+        else:
+            print(f"Fixing $ref sibling annotations in: {args.yamldir}")
+
+        reports = fix_yaml_dir(args.yamldir, dry_run=args.dry_run)
+
+        total_wrapped = 0
+        files_changed = 0
+        for report in reports:
+            print(report["path"])
+            if report["skipped_file"]:
+                print(f"  skipped file ({report['skipped_file']})")
+                continue
+            if not report["changes"]:
+                print("  no $ref properties found")
+                continue
+            for prop_name, action, preserved in report["changes"]:
+                if action == WRAPPED:
+                    total_wrapped += 1
+                    print(f"  wrapped: {prop_name} (preserved: {', '.join(preserved)})")
+                else:
+                    print(f"  skipped ({SKIP_REASONS[action]}): {prop_name}")
+            if report["rewritten"] or (args.dry_run and any(c[1] == WRAPPED for c in report["changes"])):
+                files_changed += 1
+
+        files_unchanged = len(reports) - files_changed
+        verb = "would be wrapped" if args.dry_run else "wrapped"
+        print(f"\nSummary: {total_wrapped} properties {verb} in {files_changed} files; "
+              f"{files_unchanged} files unchanged.")
+
+        # Wrapping is what exposes null 'description' placeholders in shared
+        # definitions to metaschema validation, so surface them here too.
+        print_null_description_warning(scan_dir_null_descriptions(args.yamldir))
+
 if __name__ == "__main__":
     main()

{gen3schemadev-2.4.0 → gen3schemadev-2.6.0}/src/gen3schemadev/converter.py

@@ -17,6 +17,8 @@ from dataclasses import dataclass, asdict
 from typing import Protocol, runtime_checkable, Dict, Any, List
 import logging
 
+from gen3schemadev.refs import wrap_ref_siblings
+
 
 logger = logging.getLogger(__name__)
 
@@ -488,7 +490,11 @@ def format_datetime(prop_dict: dict) -> dict:
     Formats a property dictionary with a 'type' of 'datetime' to use a $ref
     to the Gen3 _definitions.yaml#/datetime definition.
 
-    If the property is not of type 'datetime', returns the property unchanged.
+    Annotations such as 'description' are preserved by wrapping the $ref in
+    an allOf list: under JSON Schema draft-04, keywords sitting as direct
+    siblings of $ref are ignored during resolution, so they must live beside
+    allOf instead. A datetime property with no annotations becomes a bare
+    $ref. If the property is not of type 'datetime', it is returned unchanged.
 
     Example input:
         {
@@ -501,7 +507,10 @@ def format_datetime(prop_dict: dict) -> dict:
     Example output:
         {
             "collection_date": {
-                "$ref": "_definitions.yaml#/datetime"
+                "description": "Date and time of collection (datetime)",
+                "allOf": [
+                    {"$ref": "_definitions.yaml#/datetime"}
+                ]
             }
         }
 
@@ -518,16 +527,17 @@ def format_datetime(prop_dict: dict) -> dict:
     try:
         if len(prop_dict) != 1:
             raise ValueError("Expected a single property dictionary")
-        
+
         first_key = next(iter(prop_dict))
         value = prop_dict[first_key]
-        formatted_props = {}
-        
-        if 'type' in value and value['type'] == 'datetime':
-            formatted_props['$ref'] = "_definitions.yaml#/datetime"
+
+        if isinstance(value, dict) and value.get('type') == 'datetime':
+            ref_prop = {k: v for k, v in value.items() if k != 'type'}
+            ref_prop['$ref'] = "_definitions.yaml#/datetime"
+            formatted_props, _ = wrap_ref_siblings(ref_prop, first_key)
         else:
             formatted_props = value
-        
+
         output = {first_key: formatted_props}
         return output
     except Exception as e:

gen3schemadev-2.6.0/src/gen3schemadev/refs.py

@@ -0,0 +1,262 @@
+"""Utilities for making ``$ref`` properties safe under JSON Schema draft-04.
+
+Gen3 compiles data dictionaries with draft-04 semantics, where any keyword
+sitting as a direct sibling of ``$ref`` is ignored during resolution. A
+property written as::
+
+    atrial_fibrillation:
+      description: "Self-reported atrial fibrillation."
+      $ref: "_definitions.yaml#/enum_yes_no"
+
+loses its ``description`` (and ``termDef``, ``term``, etc.) when resolved by
+strict tooling such as dictionaryutils, so the data-dictionary viewer shows
+"No Description". The fix is structural: keep the annotations at the top
+level and move the ``$ref`` into an ``allOf`` list::
+
+    atrial_fibrillation:
+      description: "Self-reported atrial fibrillation."
+      allOf:
+      - $ref: "_definitions.yaml#/enum_yes_no"
+
+This module is the single source of truth for that transformation. It is
+used by the schema generator (``converter.format_datetime``), the rule
+validator, and the ``gen3schemadev fix-refs`` CLI command.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+# Actions recorded for each property examined by the transform.
+WRAPPED = "wrapped"
+BARE_REF = "bare-ref"
+ALREADY_WRAPPED = "already-wrapped"
+NO_REF = "no-ref"
+PROPERTIES_MERGE = "properties-merge"
+
+# Human-readable reasons for skipped properties, keyed by action.
+SKIP_REASONS = {
+    BARE_REF: "bare $ref, no siblings to preserve",
+    ALREADY_WRAPPED: "already wrapped in allOf/anyOf/oneOf",
+    PROPERTIES_MERGE: "$ref key in 'properties' is a Gen3 properties-merge",
+}
+
+_COMBINATORS = ("allOf", "anyOf", "oneOf")
+
+
+def has_ref(prop) -> bool:
+    """
+    Return True if a property definition references another schema, either
+    via a top-level ``$ref`` or via a ``$ref`` inside an allOf/anyOf/oneOf
+    list item.
+    """
+    if not isinstance(prop, dict):
+        return False
+    if "$ref" in prop:
+        return True
+    for combinator in _COMBINATORS:
+        items = prop.get(combinator)
+        if isinstance(items, list):
+            if any(isinstance(item, dict) and "$ref" in item for item in items):
+                return True
+    return False
+
+
+def wrap_ref_siblings(prop, prop_name: str = "") -> "tuple":
+    """
+    Rewrite a single property definition so sibling annotations survive
+    draft-04 ``$ref`` resolution.
+
+    Returns a ``(new_prop, action)`` tuple. The transformation only fires
+    when the property dict has a top-level ``$ref`` alongside at least one
+    other key; in that case the siblings keep their original order and the
+    ``$ref`` moves into an ``allOf`` list appended last. All other shapes
+    are returned unchanged:
+
+    - not a dict, or no ``$ref`` anywhere -> ``NO_REF``
+    - ``$ref`` only inside allOf/anyOf/oneOf -> ``ALREADY_WRAPPED`` (idempotent)
+    - ``{"$ref": ...}`` with no siblings -> ``BARE_REF`` (nothing is lost)
+    """
+    if not isinstance(prop, dict) or not has_ref(prop):
+        return prop, NO_REF
+
+    if "$ref" not in prop:
+        return prop, ALREADY_WRAPPED
+
+    if len(prop) == 1:
+        return prop, BARE_REF
+
+    new_prop = {k: v for k, v in prop.items() if k != "$ref"}
+    if isinstance(new_prop.get("allOf"), list):
+        # Rare edge: the property already has an allOf next to the $ref.
+        # Append rather than overwrite so existing constraints are kept.
+        new_prop["allOf"] = new_prop["allOf"] + [{"$ref": prop["$ref"]}]
+    else:
+        new_prop["allOf"] = [{"$ref": prop["$ref"]}]
+    logger.debug(f"Wrapped $ref of property '{prop_name}' in allOf.")
+    return new_prop, WRAPPED
+
+
+def fix_schema(schema: dict) -> "tuple":
+    """
+    Apply ``wrap_ref_siblings`` to every property of a node schema.
+
+    Only the direct values of ``schema["properties"]`` are examined:
+    sibling-``$ref`` annotations occur at the property level in Gen3 node
+    schemas, and a shallow walk cannot accidentally rewrite the Gen3
+    properties-merge construct (a ``$ref`` *key* directly inside the
+    ``properties`` block), which is recorded as ``PROPERTIES_MERGE`` and
+    left untouched.
+
+    Returns ``(new_schema, changes)`` where changes is a list of
+    ``(prop_name, action, preserved_keys)`` tuples. ``preserved_keys`` names
+    the sibling annotation keys kept for ``WRAPPED`` entries and is an empty
+    tuple otherwise.
+    """
+    changes = []
+    props = schema.get("properties") if isinstance(schema, dict) else None
+    if not isinstance(props, dict):
+        return schema, changes
+
+    new_props = {}
+    for prop_name, prop in props.items():
+        if prop_name == "$ref":
+            new_props[prop_name] = prop
+            changes.append((prop_name, PROPERTIES_MERGE, ()))
+            continue
+        new_prop, action = wrap_ref_siblings(prop, prop_name)
+        new_props[prop_name] = new_prop
+        preserved = ()
+        if action == WRAPPED:
+            preserved = tuple(k for k in prop.keys() if k != "$ref")
+        if action != NO_REF:
+            changes.append((prop_name, action, preserved))
+
+    new_schema = dict(schema)
+    new_schema["properties"] = new_props
+    return new_schema, changes
+
+
+def find_null_descriptions(node, path: str = "") -> list:
+    """
+    Recursively find every ``description`` key whose value is null.
+
+    The Gen3 metaschema requires ``description`` to be a string, so a
+    ``description: null`` placeholder (common in generated shared
+    definitions) is invalid. With a direct ``$ref`` the resolver happens to
+    mask it — the referencing property's own description merges over the
+    definition's null — but a bare or allOf-wrapped ref exposes the null in
+    the resolved node schema, where metaschema validation fails far away
+    from the definition that caused it.
+
+    Returns a list of dotted paths to each offender, with list items shown
+    as ``[i]`` (e.g. ``"enum_yes_no.description"``,
+    ``"properties.status.anyOf[0].description"``). Non-dict/list input
+    yields an empty list.
+    """
+    hits = []
+    if isinstance(node, dict):
+        for key, value in node.items():
+            key_path = f"{path}.{key}" if path else str(key)
+            if key == "description" and value is None:
+                hits.append(key_path)
+            else:
+                hits.extend(find_null_descriptions(value, key_path))
+    elif isinstance(node, list):
+        for i, item in enumerate(node):
+            hits.extend(find_null_descriptions(item, f"{path}[{i}]"))
+    return hits
+
+
+def scan_dir_null_descriptions(yaml_dir: str) -> list:
+    """
+    Scan every YAML file under ``yaml_dir`` (recursively) for null-valued
+    ``description`` keys.
+
+    Unlike ``fix_yaml_dir``, underscore files (``_definitions.yaml``,
+    ``_terms.yaml``, ...) are INCLUDED — shared definitions are where the
+    null placeholders usually live, and where they do the most damage
+    because every referencing property inherits them on resolution.
+
+    Returns ``"relpath: dotted.path"`` strings, one per offender.
+    """
+    from gen3schemadev.utils import load_yaml
+
+    hits = []
+    for root, _dirs, files in os.walk(yaml_dir):
+        for fname in sorted(files):
+            if not (fnmatch.fnmatch(fname, "*.yaml") or fnmatch.fnmatch(fname, "*.yml")):
+                continue
+            path = os.path.join(root, fname)
+            rel_path = os.path.relpath(path, yaml_dir)
+            schema = load_yaml(path)
+            for hit in find_null_descriptions(schema):
+                hits.append(f"{rel_path}: {hit}")
+    return hits
+
+
+def fix_yaml_dir(yaml_dir: str, dry_run: bool = False) -> list:
+    """
+    Rewrite every YAML node schema under ``yaml_dir`` (recursively) so that
+    ``$ref`` sibling annotations survive draft-04 resolution.
+
+    Files whose basename starts with an underscore (``_definitions.yaml``,
+    ``_terms.yaml``, ``_settings.yaml``) are skipped entirely: adding
+    structure or descriptions to shared definitions would force them onto
+    every referencing property. Files with no changes are never rewritten,
+    which keeps the operation idempotent. With ``dry_run=True`` nothing is
+    written at all.
+
+    Returns a list of per-file report dicts::
+
+        {"path": <relative path>, "skipped_file": <reason or None>,
+         "changes": [(prop_name, action, preserved_keys), ...],
+         "rewritten": <bool>}
+    """
+    from gen3schemadev.utils import load_yaml, write_yaml
+
+    reports = []
+    for root, _dirs, files in os.walk(yaml_dir):
+        for fname in sorted(files):
+            if not (fnmatch.fnmatch(fname, "*.yaml") or fnmatch.fnmatch(fname, "*.yml")):
+                continue
+            path = os.path.join(root, fname)
+            rel_path = os.path.relpath(path, yaml_dir)
+
+            if fname.startswith("_"):
+                reports.append({
+                    "path": rel_path,
+                    "skipped_file": "definitions/terms/settings file",
+                    "changes": [],
+                    "rewritten": False,
+                })
+                continue
+
+            schema = load_yaml(path)
+            if not isinstance(schema, dict):
+                reports.append({
+                    "path": rel_path,
+                    "skipped_file": "not a mapping, no properties to fix",
+                    "changes": [],
+                    "rewritten": False,
+                })
+                continue
+
+            new_schema, changes = fix_schema(schema)
+            wrapped = [c for c in changes if c[1] == WRAPPED]
+            rewritten = False
+            if wrapped and not dry_run:
+                write_yaml(new_schema, path)
+                rewritten = True
+
+            reports.append({
+                "path": rel_path,
+                "skipped_file": None,
+                "changes": changes,
+                "rewritten": rewritten,
+            })
+    return reports

{gen3schemadev-2.4.0 → gen3schemadev-2.6.0}/src/gen3schemadev/schema/schema_templates/program.yaml

@@ -24,8 +24,9 @@ properties:
   type:
     type: string
   id:
-    $ref: _definitions.yaml#/UUID
     systemAlias: node_id
+    allOf:
+    - $ref: _definitions.yaml#/UUID
   name:
     type: string
     description: Full name/title of the program.

{gen3schemadev-2.4.0 → gen3schemadev-2.6.0}/src/gen3schemadev/schema/schema_templates/project.yaml

@@ -35,9 +35,10 @@ properties:
   type:
     type: string
   id:
-    $ref: _definitions.yaml#/UUID
     systemAlias: node_id
     description: UUID for the project.
+    allOf:
+    - $ref: _definitions.yaml#/UUID
   name:
     type: string
     description: Display name/brief description for the project.
@@ -62,10 +63,11 @@ properties:
     description: The ID of the source providing support/grant resources.
     type: string
   programs:
-    $ref: _definitions.yaml#/to_one
     description: 'Indicates that the Study is logically part of the indicated Study.
 
       '
+    allOf:
+    - $ref: _definitions.yaml#/to_one
   state:
     description: 'The possible states a project can be in.  All but `open` are
 

{gen3schemadev-2.4.0 → gen3schemadev-2.6.0}/src/gen3schemadev/validators/rule_validator.py

@@ -1,6 +1,7 @@
 # validates gen3 bundled jsonschema (.json) by testing gen3 specific business rules
 import logging
 from gen3schemadev.converter import link_suffix
+from gen3schemadev.refs import has_ref
 
 logger = logging.getLogger(__name__)
 
@@ -188,10 +189,12 @@ class RuleValidator:
                 )
                 continue
 
-            # Skip if property definition contains a $ref (i.e., is an alias/reference)
-            if "$ref" in value:
+            # Skip if property definition contains a $ref (i.e., is an alias/reference),
+            # either top-level or wrapped in allOf/anyOf/oneOf
+            if has_ref(value):
                 logger.debug(
-                    f"Skipping property '{key}' because it contains a '$ref'."
+                    f"Skipping property '{key}' because it contains a '$ref' "
+                    "(top-level or inside allOf/anyOf/oneOf)."
                 )
                 continue