Gen3SchemaDev 2.3.6__tar.gz → 2.5.0__tar.gz

{gen3schemadev-2.3.6 → gen3schemadev-2.5.0}/PKG-INFO

@@ -1,14 +1,17 @@
 Metadata-Version: 2.4
 Name: Gen3SchemaDev
-Version: 2.3.6
+Version: 2.5.0
 Summary: Tool for data modelling in Gen3
 License: Apache 2.0
 License-File: LICENSE
 Author: JoshuaHarris391
 Author-email: harjo391@gmail.com
-Requires-Python: >=3.12.10,<4.0
+Requires-Python: >=3.9.5,<4.0
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: check-jsonschema (>=0.34.1,<0.35.0)
@@ -32,6 +35,17 @@ This repository aims to provide the documentation, learning materials, and softw
 - [Guide to creating your first dictionary](docs/gen3schemadev/first_dictionary.md)
 - [Troubleshooting](docs/gen3schemadev/troubleshooting.md)
 
+### Fixing `$ref` properties in an existing dictionary
+
+JSON Schema draft-04 (which Gen3 uses) ignores any keyword sitting as a direct sibling of `$ref`, so properties written as `{description: ..., $ref: _definitions.yaml#/X}` lose their description in the data-dictionary viewer. The `fix-refs` command rewrites such properties in place, moving the `$ref` into an `allOf` list so annotations survive resolution:
+
+```bash
+gen3schemadev fix-refs -y path/to/dictionary --dry-run   # preview changes
+gen3schemadev fix-refs -y path/to/dictionary             # rewrite in place
+```
+
+The command is idempotent and never touches bare refs, refs already inside `allOf`/`anyOf`/`oneOf`, the `properties: {$ref: ...}` merge construct, or `_definitions.yaml`/`_terms.yaml`/`_settings.yaml`.
+
 
 ## Deep dive into Gen3 Data Modelling
 *Special Thanks to Marion Shadbolt for providing the [source material](https://github.com/AustralianBioCommons/umccr-dictionary/tree/main/docs/schemas)*

{gen3schemadev-2.3.6 → gen3schemadev-2.5.0}/README.md

@@ -12,6 +12,17 @@ This repository aims to provide the documentation, learning materials, and softw
 - [Guide to creating your first dictionary](docs/gen3schemadev/first_dictionary.md)
 - [Troubleshooting](docs/gen3schemadev/troubleshooting.md)
 
+### Fixing `$ref` properties in an existing dictionary
+
+JSON Schema draft-04 (which Gen3 uses) ignores any keyword sitting as a direct sibling of `$ref`, so properties written as `{description: ..., $ref: _definitions.yaml#/X}` lose their description in the data-dictionary viewer. The `fix-refs` command rewrites such properties in place, moving the `$ref` into an `allOf` list so annotations survive resolution:
+
+```bash
+gen3schemadev fix-refs -y path/to/dictionary --dry-run   # preview changes
+gen3schemadev fix-refs -y path/to/dictionary             # rewrite in place
+```
+
+The command is idempotent and never touches bare refs, refs already inside `allOf`/`anyOf`/`oneOf`, the `properties: {$ref: ...}` merge construct, or `_definitions.yaml`/`_terms.yaml`/`_settings.yaml`.
+
 
 ## Deep dive into Gen3 Data Modelling
 *Special Thanks to Marion Shadbolt for providing the [source material](https://github.com/AustralianBioCommons/umccr-dictionary/tree/main/docs/schemas)*

{gen3schemadev-2.3.6 → gen3schemadev-2.5.0}/pyproject.toml

@@ -1,13 +1,13 @@
 [project]
 name = "Gen3SchemaDev"
-version = "2.3.6"
+version = "2.5.0"
 description = "Tool for data modelling in Gen3"
 authors = [
     {name = "JoshuaHarris391",email = "harjo391@gmail.com"}
 ]
 license = {text = "Apache 2.0"}
 readme = "README.md"
-requires-python = ">=3.12.10,<4.0"
+requires-python = ">=3.9.5,<4.0"
 dependencies = [
     "setuptools>=70.0",
     "gen3-validator (>=2.0.0,<3.0.0)",

{gen3schemadev-2.3.6 → gen3schemadev-2.5.0}/src/gen3schemadev/cli.py

@@ -20,6 +20,7 @@ from gen3schemadev.validators.metaschema_validator import validate_schema_with_m
 from importlib.metadata import version
 from gen3schemadev.ddvis import visualise_with_docker
 from gen3schemadev.validators.rule_validator import RuleValidator
+from gen3schemadev.refs import WRAPPED, SKIP_REASONS, fix_yaml_dir
 
 
 def main():
@@ -142,7 +143,28 @@ def main():
         action="store_true",
         help="Set logging level to DEBUG"
     )
-    
+
+    # Create 'fix-refs' subcommand
+    fixrefs_parser = subparsers.add_parser(
+        "fix-refs",
+        help="Rewrite properties with '$ref' siblings into draft-04-safe allOf form (in place)"
+    )
+    fixrefs_parser.add_argument(
+        "-y", "--yamldir",
+        required=True,
+        help="Directory of Gen3 YAML files to rewrite in place (recursive)"
+    )
+    fixrefs_parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Report what would change without writing files"
+    )
+    fixrefs_parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Set logging level to DEBUG"
+    )
+
     args = parser.parse_args()
     
     # Handle case where no command is provided
@@ -306,5 +328,41 @@ def main():
         output_path = args.output or "input_example.yaml"
         write_yaml(init_yaml, output_path)
 
+    elif args.command == "fix-refs":
+        if not os.path.isdir(args.yamldir):
+            print(f"ERROR: Directory not found: {args.yamldir}", file=sys.stderr)
+            sys.exit(1)
+
+        if args.dry_run:
+            print(f"Dry run: scanning {args.yamldir} (no files will be written)")
+        else:
+            print(f"Fixing $ref sibling annotations in: {args.yamldir}")
+
+        reports = fix_yaml_dir(args.yamldir, dry_run=args.dry_run)
+
+        total_wrapped = 0
+        files_changed = 0
+        for report in reports:
+            print(report["path"])
+            if report["skipped_file"]:
+                print(f"  skipped file ({report['skipped_file']})")
+                continue
+            if not report["changes"]:
+                print("  no $ref properties found")
+                continue
+            for prop_name, action, preserved in report["changes"]:
+                if action == WRAPPED:
+                    total_wrapped += 1
+                    print(f"  wrapped: {prop_name} (preserved: {', '.join(preserved)})")
+                else:
+                    print(f"  skipped ({SKIP_REASONS[action]}): {prop_name}")
+            if report["rewritten"] or (args.dry_run and any(c[1] == WRAPPED for c in report["changes"])):
+                files_changed += 1
+
+        files_unchanged = len(reports) - files_changed
+        verb = "would be wrapped" if args.dry_run else "wrapped"
+        print(f"\nSummary: {total_wrapped} properties {verb} in {files_changed} files; "
+              f"{files_unchanged} files unchanged.")
+
 if __name__ == "__main__":
     main()

{gen3schemadev-2.3.6 → gen3schemadev-2.5.0}/src/gen3schemadev/converter.py

@@ -11,10 +11,14 @@ Expected Data Structure:
     - links: List of link objects with attributes: child, parent, multiplicity
 """
 
+from __future__ import annotations
+
 from dataclasses import dataclass, asdict
 from typing import Protocol, runtime_checkable, Dict, Any, List
 import logging
 
+from gen3schemadev.refs import wrap_ref_siblings
+
 
 logger = logging.getLogger(__name__)
 
@@ -486,7 +490,11 @@ def format_datetime(prop_dict: dict) -> dict:
     Formats a property dictionary with a 'type' of 'datetime' to use a $ref
     to the Gen3 _definitions.yaml#/datetime definition.
 
-    If the property is not of type 'datetime', returns the property unchanged.
+    Annotations such as 'description' are preserved by wrapping the $ref in
+    an allOf list: under JSON Schema draft-04, keywords sitting as direct
+    siblings of $ref are ignored during resolution, so they must live beside
+    allOf instead. A datetime property with no annotations becomes a bare
+    $ref. If the property is not of type 'datetime', it is returned unchanged.
 
     Example input:
         {
@@ -499,7 +507,10 @@ def format_datetime(prop_dict: dict) -> dict:
     Example output:
         {
             "collection_date": {
-                "$ref": "_definitions.yaml#/datetime"
+                "description": "Date and time of collection (datetime)",
+                "allOf": [
+                    {"$ref": "_definitions.yaml#/datetime"}
+                ]
             }
         }
 
@@ -516,16 +527,17 @@ def format_datetime(prop_dict: dict) -> dict:
     try:
         if len(prop_dict) != 1:
             raise ValueError("Expected a single property dictionary")
-        
+
         first_key = next(iter(prop_dict))
         value = prop_dict[first_key]
-        formatted_props = {}
-        
-        if 'type' in value and value['type'] == 'datetime':
-            formatted_props['$ref'] = "_definitions.yaml#/datetime"
+
+        if isinstance(value, dict) and value.get('type') == 'datetime':
+            ref_prop = {k: v for k, v in value.items() if k != 'type'}
+            ref_prop['$ref'] = "_definitions.yaml#/datetime"
+            formatted_props, _ = wrap_ref_siblings(ref_prop, first_key)
         else:
             formatted_props = value
-        
+
         output = {first_key: formatted_props}
         return output
     except Exception as e:

gen3schemadev-2.5.0/src/gen3schemadev/refs.py

@@ -0,0 +1,204 @@
+"""Utilities for making ``$ref`` properties safe under JSON Schema draft-04.
+
+Gen3 compiles data dictionaries with draft-04 semantics, where any keyword
+sitting as a direct sibling of ``$ref`` is ignored during resolution. A
+property written as::
+
+    atrial_fibrillation:
+      description: "Self-reported atrial fibrillation."
+      $ref: "_definitions.yaml#/enum_yes_no"
+
+loses its ``description`` (and ``termDef``, ``term``, etc.) when resolved by
+strict tooling such as dictionaryutils, so the data-dictionary viewer shows
+"No Description". The fix is structural: keep the annotations at the top
+level and move the ``$ref`` into an ``allOf`` list::
+
+    atrial_fibrillation:
+      description: "Self-reported atrial fibrillation."
+      allOf:
+      - $ref: "_definitions.yaml#/enum_yes_no"
+
+This module is the single source of truth for that transformation. It is
+used by the schema generator (``converter.format_datetime``), the rule
+validator, and the ``gen3schemadev fix-refs`` CLI command.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+# Actions recorded for each property examined by the transform.
+WRAPPED = "wrapped"
+BARE_REF = "bare-ref"
+ALREADY_WRAPPED = "already-wrapped"
+NO_REF = "no-ref"
+PROPERTIES_MERGE = "properties-merge"
+
+# Human-readable reasons for skipped properties, keyed by action.
+SKIP_REASONS = {
+    BARE_REF: "bare $ref, no siblings to preserve",
+    ALREADY_WRAPPED: "already wrapped in allOf/anyOf/oneOf",
+    PROPERTIES_MERGE: "$ref key in 'properties' is a Gen3 properties-merge",
+}
+
+_COMBINATORS = ("allOf", "anyOf", "oneOf")
+
+
+def has_ref(prop) -> bool:
+    """
+    Return True if a property definition references another schema, either
+    via a top-level ``$ref`` or via a ``$ref`` inside an allOf/anyOf/oneOf
+    list item.
+    """
+    if not isinstance(prop, dict):
+        return False
+    if "$ref" in prop:
+        return True
+    for combinator in _COMBINATORS:
+        items = prop.get(combinator)
+        if isinstance(items, list):
+            if any(isinstance(item, dict) and "$ref" in item for item in items):
+                return True
+    return False
+
+
+def wrap_ref_siblings(prop, prop_name: str = "") -> "tuple":
+    """
+    Rewrite a single property definition so sibling annotations survive
+    draft-04 ``$ref`` resolution.
+
+    Returns a ``(new_prop, action)`` tuple. The transformation only fires
+    when the property dict has a top-level ``$ref`` alongside at least one
+    other key; in that case the siblings keep their original order and the
+    ``$ref`` moves into an ``allOf`` list appended last. All other shapes
+    are returned unchanged:
+
+    - not a dict, or no ``$ref`` anywhere -> ``NO_REF``
+    - ``$ref`` only inside allOf/anyOf/oneOf -> ``ALREADY_WRAPPED`` (idempotent)
+    - ``{"$ref": ...}`` with no siblings -> ``BARE_REF`` (nothing is lost)
+    """
+    if not isinstance(prop, dict) or not has_ref(prop):
+        return prop, NO_REF
+
+    if "$ref" not in prop:
+        return prop, ALREADY_WRAPPED
+
+    if len(prop) == 1:
+        return prop, BARE_REF
+
+    new_prop = {k: v for k, v in prop.items() if k != "$ref"}
+    if isinstance(new_prop.get("allOf"), list):
+        # Rare edge: the property already has an allOf next to the $ref.
+        # Append rather than overwrite so existing constraints are kept.
+        new_prop["allOf"] = new_prop["allOf"] + [{"$ref": prop["$ref"]}]
+    else:
+        new_prop["allOf"] = [{"$ref": prop["$ref"]}]
+    logger.debug(f"Wrapped $ref of property '{prop_name}' in allOf.")
+    return new_prop, WRAPPED
+
+
+def fix_schema(schema: dict) -> "tuple":
+    """
+    Apply ``wrap_ref_siblings`` to every property of a node schema.
+
+    Only the direct values of ``schema["properties"]`` are examined:
+    sibling-``$ref`` annotations occur at the property level in Gen3 node
+    schemas, and a shallow walk cannot accidentally rewrite the Gen3
+    properties-merge construct (a ``$ref`` *key* directly inside the
+    ``properties`` block), which is recorded as ``PROPERTIES_MERGE`` and
+    left untouched.
+
+    Returns ``(new_schema, changes)`` where changes is a list of
+    ``(prop_name, action, preserved_keys)`` tuples. ``preserved_keys`` names
+    the sibling annotation keys kept for ``WRAPPED`` entries and is an empty
+    tuple otherwise.
+    """
+    changes = []
+    props = schema.get("properties") if isinstance(schema, dict) else None
+    if not isinstance(props, dict):
+        return schema, changes
+
+    new_props = {}
+    for prop_name, prop in props.items():
+        if prop_name == "$ref":
+            new_props[prop_name] = prop
+            changes.append((prop_name, PROPERTIES_MERGE, ()))
+            continue
+        new_prop, action = wrap_ref_siblings(prop, prop_name)
+        new_props[prop_name] = new_prop
+        preserved = ()
+        if action == WRAPPED:
+            preserved = tuple(k for k in prop.keys() if k != "$ref")
+        if action != NO_REF:
+            changes.append((prop_name, action, preserved))
+
+    new_schema = dict(schema)
+    new_schema["properties"] = new_props
+    return new_schema, changes
+
+
+def fix_yaml_dir(yaml_dir: str, dry_run: bool = False) -> list:
+    """
+    Rewrite every YAML node schema under ``yaml_dir`` (recursively) so that
+    ``$ref`` sibling annotations survive draft-04 resolution.
+
+    Files whose basename starts with an underscore (``_definitions.yaml``,
+    ``_terms.yaml``, ``_settings.yaml``) are skipped entirely: adding
+    structure or descriptions to shared definitions would force them onto
+    every referencing property. Files with no changes are never rewritten,
+    which keeps the operation idempotent. With ``dry_run=True`` nothing is
+    written at all.
+
+    Returns a list of per-file report dicts::
+
+        {"path": <relative path>, "skipped_file": <reason or None>,
+         "changes": [(prop_name, action, preserved_keys), ...],
+         "rewritten": <bool>}
+    """
+    from gen3schemadev.utils import load_yaml, write_yaml
+
+    reports = []
+    for root, _dirs, files in os.walk(yaml_dir):
+        for fname in sorted(files):
+            if not (fnmatch.fnmatch(fname, "*.yaml") or fnmatch.fnmatch(fname, "*.yml")):
+                continue
+            path = os.path.join(root, fname)
+            rel_path = os.path.relpath(path, yaml_dir)
+
+            if fname.startswith("_"):
+                reports.append({
+                    "path": rel_path,
+                    "skipped_file": "definitions/terms/settings file",
+                    "changes": [],
+                    "rewritten": False,
+                })
+                continue
+
+            schema = load_yaml(path)
+            if not isinstance(schema, dict):
+                reports.append({
+                    "path": rel_path,
+                    "skipped_file": "not a mapping, no properties to fix",
+                    "changes": [],
+                    "rewritten": False,
+                })
+                continue
+
+            new_schema, changes = fix_schema(schema)
+            wrapped = [c for c in changes if c[1] == WRAPPED]
+            rewritten = False
+            if wrapped and not dry_run:
+                write_yaml(new_schema, path)
+                rewritten = True
+
+            reports.append({
+                "path": rel_path,
+                "skipped_file": None,
+                "changes": changes,
+                "rewritten": rewritten,
+            })
+    return reports

{gen3schemadev-2.3.6 → gen3schemadev-2.5.0}/src/gen3schemadev/schema/schema_templates/program.yaml

@@ -24,8 +24,9 @@ properties:
   type:
     type: string
   id:
-    $ref: _definitions.yaml#/UUID
     systemAlias: node_id
+    allOf:
+    - $ref: _definitions.yaml#/UUID
   name:
     type: string
     description: Full name/title of the program.

{gen3schemadev-2.3.6 → gen3schemadev-2.5.0}/src/gen3schemadev/schema/schema_templates/project.yaml

@@ -35,9 +35,10 @@ properties:
   type:
     type: string
   id:
-    $ref: _definitions.yaml#/UUID
     systemAlias: node_id
     description: UUID for the project.
+    allOf:
+    - $ref: _definitions.yaml#/UUID
   name:
     type: string
     description: Display name/brief description for the project.
@@ -62,10 +63,11 @@ properties:
     description: The ID of the source providing support/grant resources.
     type: string
   programs:
-    $ref: _definitions.yaml#/to_one
     description: 'Indicates that the Study is logically part of the indicated Study.
 
       '
+    allOf:
+    - $ref: _definitions.yaml#/to_one
   state:
     description: 'The possible states a project can be in.  All but `open` are
 

{gen3schemadev-2.3.6 → gen3schemadev-2.5.0}/src/gen3schemadev/validators/rule_validator.py

@@ -1,6 +1,7 @@
 # validates gen3 bundled jsonschema (.json) by testing gen3 specific business rules
 import logging
 from gen3schemadev.converter import link_suffix
+from gen3schemadev.refs import has_ref
 
 logger = logging.getLogger(__name__)
 
@@ -188,10 +189,12 @@ class RuleValidator:
                 )
                 continue
 
-            # Skip if property definition contains a $ref (i.e., is an alias/reference)
-            if "$ref" in value:
+            # Skip if property definition contains a $ref (i.e., is an alias/reference),
+            # either top-level or wrapped in allOf/anyOf/oneOf
+            if has_ref(value):
                 logger.debug(
-                    f"Skipping property '{key}' because it contains a '$ref'."
+                    f"Skipping property '{key}' because it contains a '$ref' "
+                    "(top-level or inside allOf/anyOf/oneOf)."
                 )
                 continue