pyegeria 5.4.3.2__py3-none-any.whl → 5.4.3.4__py3-none-any.whl

md_processing/md_commands/data_designer_commands.py

@@ -11,8 +11,22 @@ from rich import print
 from rich.console import Console
 from rich.markdown import Markdown
 
-from md_processing.md_processing_utils.common_md_proc_utils import (parse_upsert_command, parse_view_command)
-from md_processing.md_processing_utils.common_md_utils import update_element_dictionary
+from md_processing.md_processing_utils.common_md_proc_utils import (parse_upsert_command, parse_view_command )
+from md_processing.md_processing_utils.common_md_utils import (
+    update_element_dictionary,
+    set_find_body,
+    set_create_body,
+    set_element_prop_body,
+    set_update_body,
+    set_rel_request_body,
+    set_update_status_body,
+    set_rel_prop_body,
+    set_data_field_body,
+)
+# Developer note: Request bodies follow a two-layer pattern:
+# - Outer: action wrapper (NewElementRequestBody/UpdateElementRequestBody/...)
+# - Inner: element-specific Properties dict placed under the outer["properties"] key
+# Always build inner bodies via helpers (set_element_prop_body, set_data_field_body, etc.) and assign to the outer
 from md_processing.md_processing_utils.extraction_utils import (extract_command_plus, update_a_command)
 from md_processing.md_processing_utils.md_processing_constants import (load_commands, ERROR)
 from pyegeria import DEBUG_LEVEL, body_slimmer
@@ -53,7 +67,7 @@ def add_member_to_data_collections(egeria_client: EgeriaTech, collection_list: l
     Add member to data dictionaries and data specifications.
     """
     body = {
-        "class": "RelationshipRequestBody", "properties": {
+        "class": "NewRelationshipRequestBody", "properties": {
             "class": "CollectionMembershipProperties", "membershipRationale": "User Specified",
             "notes": "Added by Dr.Egeria"
             }
@@ -417,6 +431,7 @@ def process_data_spec_upsert_command(egeria_client: EgeriaTech, txt: str, direct
     """
 
     command, object_type, object_action = extract_command_plus(txt)
+    print(Markdown(f"# {command}\n"))
 
     parsed_output = parse_upsert_command(egeria_client, object_type, object_action, txt, directive)
 
@@ -426,34 +441,13 @@ def process_data_spec_upsert_command(egeria_client: EgeriaTech, txt: str, direct
     qualified_name = parsed_output.get('qualified_name', None)
     guid = parsed_output.get('guid', None)
 
-    print(Markdown(parsed_output['display']))
-
     logger.debug(json.dumps(parsed_output, indent=4))
 
     attributes = parsed_output['attributes']
-    description = attributes['Description'].get('value', None)
-    display_name = attributes['Display Name'].get('value', None)
-    anchor_guid = attributes.get('Anchor ID', {}).get('guid', None)
-    parent_guid = attributes.get('Parent ID', {}).get('guid', None)
-    parent_relationship_type_name = attributes.get('Parent Relationship Type Name', {}).get('value',
-                                                                                            "CollectionMembership")
-    parent_at_end1 = attributes.get('Parent at End1', {}).get('value', True)
-
-    anchor_scope_guid = attributes.get('Anchor Scope GUID', {}).get('value', None)
-    is_own_anchor = attributes.get('Is Own Anchor', {}).get('value', True)
-    if parent_guid is None:
-        is_own_anchor = True
-
-    collection_type = attributes.get('Collection Type', {}).get('value', None)
-
-    replace_all_props = not attributes.get('Merge Update', {}).get('value', True)
-
-    additional_prop = attributes.get('Additional Properties', {}).get('value', None)
-    additional_properties = json.loads(additional_prop) if additional_prop is not None else None
-    extended_prop = attributes.get('Extended Properties', {}).get('value', None)
-    extended_properties = json.loads(extended_prop) if extended_prop is not None else None
+    display_name = attributes.get('Display Name', {}).get('value', "None Found")
+    status = attributes.get('Status', {}).get('value', None)
+    print(Markdown(parsed_output['display']))
 
-    replace_all_props = not attributes.get('Merge Update', {}).get('value', True)
     in_data_spec_list = attributes.get('In Data Specification', {}).get('value', None)
     in_data_spec_valid = attributes.get('In Data Specification', {}).get('valid', None)
     in_data_spec_exists = attributes.get('In Data Specification', {}).get('exists', None)
@@ -469,6 +463,7 @@ def process_data_spec_upsert_command(egeria_client: EgeriaTech, txt: str, direct
         return valid
 
     elif directive == "process":
+
         try:
             if object_action == "Update":
                 if not exists:
@@ -482,17 +477,20 @@ def process_data_spec_upsert_command(egeria_client: EgeriaTech, txt: str, direct
                     print(Markdown(
                         f"==> Validation of {command} completed successfully! Proceeding to apply the changes.\n"))
 
-                egeria_client.update_collection(guid, qualified_name, display_name, description, collection_type,
-                                                 additional_properties,
-                                                extended_properties, replace_all_props)
+                body = set_update_body(object_type, attributes)
+                body['properties'] = set_element_prop_body("Data Spec", qualified_name, attributes)
+
+                egeria_client.update_collection(guid, body)
+                if status:
+                    egeria_client.update_collection_status(guid, status)
+
                 logger.success(f"Updated  {object_type} `{display_name}` with GUID {guid}\n\n___")
                 update_element_dictionary(qualified_name, {
                     'guid': guid, 'display_name': display_name
-                    })
-                return egeria_client.get_collection_by_guid(guid, collection_type='Data Specification',
+                })
+                return egeria_client.get_collection_by_guid(guid, element_type='Data Specification',
                                                             output_format='MD')
 
-
             elif object_action == "Create":
                 if valid is False and exists:
                     msg = (f"  Data Specification `{display_name}` already exists and result document updated changing "
@@ -505,24 +503,27 @@ def process_data_spec_upsert_command(egeria_client: EgeriaTech, txt: str, direct
                     logger.error(msg)
                     return None
                 else:
-                    guid = egeria_client.create_data_spec_collection(display_name, description, qualified_name,
-                                                                     is_own_anchor, anchor_guid, parent_guid,
-                                                                     parent_relationship_type_name, parent_at_end1,
-                                                                     collection_type, anchor_scope_guid,
-                                                                     additional_properties, extended_properties)
+                    body = set_create_body(object_type, attributes)
+                    body["properties"] = set_element_prop_body(object_type, qualified_name, attributes)
+                    parent_guid = body.get('parentGuid', None)
+                    if parent_guid:
+                        body['parentRelationshipTypeName'] = "CollectionMembership"
+                        body['parentAtEnd1'] = True
+
+                    guid = egeria_client.create_collection(body=body)
                     if guid:
                         update_element_dictionary(qualified_name, {
                             'guid': guid, 'display_name': display_name
-                            })
+                        })
                         msg = f"Created Element `{display_name}` with GUID {guid}\n\n___"
                         logger.success(msg)
-                        return egeria_client.get_collection_by_guid(guid, collection_type='Data Specification',
-                                                                    output_format='MD')
+                        return egeria_client.get_collection_by_guid(guid, object_type, output_format='MD')
                     else:
                         msg = f"Failed to create element `{display_name}` with GUID {guid}\n\n___"
                         logger.error(msg)
                         return None
 
+
         except Exception as e:
             logger.error(f"Error performing {command}: {e}")
             return None
@@ -542,7 +543,9 @@ def process_data_dict_upsert_command(egeria_client: EgeriaTech, txt: str, direct
     :return: A string summarizing the outcome of the processing.
     """
 
+
     command, object_type, object_action = extract_command_plus(txt)
+    print(Markdown(f"# {command}\n"))
 
     parsed_output = parse_upsert_command(egeria_client, object_type, object_action, txt, directive)
 
@@ -553,31 +556,15 @@ def process_data_dict_upsert_command(egeria_client: EgeriaTech, txt: str, direct
     guid = parsed_output.get('guid', None)
 
     print(Markdown(parsed_output['display']))
+
     logger.debug(json.dumps(parsed_output, indent=4))
 
     attributes = parsed_output['attributes']
-    description = attributes['Description'].get('value', None)
     display_name = attributes.get('Display Name', {}).get('value', "None Found")
-    display_name = display_name if display_name is not None else "None Found"
-    anchor_guid = attributes.get('Anchor ID', {}).get('guid', None)
-    parent_guid = attributes.get('Parent ID', {}).get('guid', None)
-    parent_relationship_type_name = attributes.get('Parent Relationship Type Name', {}).get('value',
-                                                                                            "CollectionMembership")
-    parent_at_end1 = attributes.get('Parent at End1', {}).get('value', True)
-
-    anchor_scope_guid = attributes.get('Anchor Scope GUID', {}).get('value', None)
-    is_own_anchor = attributes.get('Is Own Anchor', {}).get('value', True)
-    if parent_guid is None:
-        is_own_anchor = True
-    collection_type = attributes.get('Collection Type', {}).get('value', None)
-    replace_all_props = not attributes.get('Merge Update', {}).get('value', True)
-
-    additional_prop = attributes.get('Additional Properties', {}).get('value', None)
-    additional_properties = json.loads(additional_prop) if additional_prop is not None else None
-    extended_prop = attributes.get('Extended Properties', {}).get('value', None)
-    extended_properties = json.loads(extended_prop) if extended_prop is not None else None
+    status = attributes.get('Status', {}).get('value', None)
 
     if directive == "display":
+
         return None
     elif directive == "validate":
         if valid:
@@ -590,62 +577,73 @@ def process_data_dict_upsert_command(egeria_client: EgeriaTech, txt: str, direct
 
         try:
             if object_action == "Update":
-
                 if not exists:
-                    logger.error(f"Element `{display_name}` does not exist! Updating result document with Create "
-                                 f"object_action\n\n___")
+                    msg = (f" Element `{display_name}` does not exist! Updating result document with Create "
+                           f"object_action\n")
+                    logger.error(msg)
                     return update_a_command(txt, object_action, object_type, qualified_name, guid)
                 elif not valid:
-                    logger.error(f"Element `{display_name}` does not have a valid specification? Review..\n\n___ ")
                     return None
                 else:
                     print(Markdown(
-                        f"==> Validation of {command} completed successfully! Proceeding to apply the changes."))
+                        f"==> Validation of {command} completed successfully! Proceeding to apply the changes.\n"))
+
+                body = set_update_body(object_type, attributes)
+                body['properties'] = set_element_prop_body("Data Spec", qualified_name, attributes)
+
+                egeria_client.update_collection(guid, body)
+                if status:
+                    egeria_client.update_collection_status(guid, status)
 
-                egeria_client.update_collection(guid, qualified_name, display_name, description, collection_type,
-                                                 additional_properties,
-                                                extended_properties, replace_all_props)
                 logger.success(f"Updated  {object_type} `{display_name}` with GUID {guid}\n\n___")
                 update_element_dictionary(qualified_name, {
                     'guid': guid, 'display_name': display_name
-                    })
-                return egeria_client.get_collection_by_guid(guid, collection_type='Data Dictionary', output_format='MD')
+                })
+                return egeria_client.get_collection_by_guid(guid, element_type='Data Specification',
+                                                            output_format='MD')
 
             elif object_action == "Create":
                 if valid is False and exists:
-                    logger.error(f"\nElement `{display_name}` already exists and result document updated changing "
-                                 f"`Create` to `Update` in processed output\n\n___")
+                    msg = (f"  Data Specification `{display_name}` already exists and result document updated changing "
+                           f"`Create` to `Update` in processed output\n\n___")
+                    logger.error(msg)
                     return update_a_command(txt, object_action, object_type, qualified_name, guid)
+                elif valid is False :
+                    msg = f" invalid data? - Correct and try again\n\n___"
+                    logger.error(msg)
+                    return None
                 else:
-                    guid = egeria_client.create_data_dictionary_collection(display_name, description, qualified_name,
-                                                                           is_own_anchor, anchor_guid, parent_guid,
-                                                                           parent_relationship_type_name,
-                                                                           parent_at_end1, collection_type,
-                                                                           anchor_scope_guid, additional_properties,
-                                                                           extended_properties)
+                    body = set_create_body(object_type, attributes)
+                    body["properties"] = set_element_prop_body(object_type, qualified_name, attributes)
+                    parent_guid = body.get('parentGuid', None)
+                    if parent_guid:
+                        body['parentRelationshipTypeName'] = "CollectionMembership"
+                        body['parentAtEnd1'] = True
+
+                    guid = egeria_client.create_collection(body=body)
                     if guid:
                         update_element_dictionary(qualified_name, {
                             'guid': guid, 'display_name': display_name
-                            })
-                        logger.success(f"Created Element `{display_name}` with GUID {guid}\n\n___")
-
-                        return egeria_client.get_collection_by_guid(guid, collection_type='Data Dictionary',
-                                                                    output_format='MD')
+                        })
+                        msg = f"Created Element `{display_name}` with GUID {guid}\n\n___"
+                        logger.success(msg)
+                        return egeria_client.get_collection_by_guid(guid, object_type, output_format='MD')
                     else:
-                        logger.error(f"Failed to create Term `{display_name}`\n\n___")
+                        msg = f"Failed to create element `{display_name}` with GUID {guid}\n\n___"
+                        logger.error(msg)
                         return None
 
+
         except Exception as e:
-            logger.error(f"{ERROR}Error performing {command}: {e}")
-            Console().print_exception(show_locals=True)
+            logger.error(f"Error performing {command}: {e}")
             return None
     else:
         return None
 
 
 @logger.catch
-def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[
-    str]:
+def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str,
+                                          directive: str = "display") -> Optional[str]:
     """
     Processes a data structure create or update object_action by extracting key attributes such as
     spec name, parent_guid, parent_relationship_type, parent_at_end_1, category
@@ -655,9 +653,7 @@ def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str, d
     :param directive: an optional string indicating the directive to be used - display, validate or execute
     :return: A string summarizing the outcome of the processing.
     """
-    from md_processing.md_processing_utils.common_md_utils import set_debug_level
 
-    set_debug_level(directive)
 
     command, object_type, object_action = extract_command_plus(txt)
     print(Markdown(f"# {command}\n"))
@@ -670,7 +666,7 @@ def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str, d
     qualified_name = parsed_output.get('qualified_name', None)
     guid = parsed_output.get('guid', None)
 
-    print(Markdown(parsed_output['display']))
+
 
     if directive == "display":
         return None
@@ -685,48 +681,16 @@ def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str, d
     elif directive == "process":
         logger.debug(json.dumps(parsed_output, indent=4))
         attributes = parsed_output['attributes']
-
-        external_source_guid = attributes.get('External Source Name', {}).get('guid', None)
-        external_source_name = attributes.get('External Source Name', {}).get('value', None)
-        effective_time = attributes.get('Effective Time', {}).get('value', None)
-        for_lineage = attributes.get('For Lineage', {}).get('value', None)
-        for_duplicate_processing = attributes.get('For Duplicate Processing', {}).get('value', None)
-        anchor_guid = attributes.get('Anchor ID', {}).get('guid', None)
-        is_own_anchor = attributes.get('Is Own Anchor', {}).get('value', None)
-        parent_id = attributes.get('Parent ID', {}).get('value', None)
-        parent_guid = attributes.get('Parent ID', {}).get('guid', None)
-        parent_relationship_type_name = attributes.get('Parent Relationship Type Name', {}).get('value', None)
-        parent_relationship_properties = attributes.get('Parent Relationship Properties', {}).get('value', None)
-        parent_at_end1 = attributes.get('Parent at End1', {}).get('value', None)
-
-        display_name = attributes['Display Name'].get('value', None)
-
-        namespace = attributes.get('Namespace', {}).get('value', None)
-        description = attributes.get('Description', {}).get('value', None)
-        version_id = attributes.get('Version Identifier', {}).get('value', None)
-        aliases = attributes.get('Aliases', {}).get('value', None)
-        name_patterns = attributes.get('Name Patterns', {}).get('value', None)
-        is_nullable = attributes.get('Is Nullable', {}).get('value', None)
-        default_value = attributes.get('Default Value', {}).get('value', None)
-        data_type = attributes.get('Data Type', {}).get('value', None)
-        min_length = attributes.get('Minimum Length', {}).get('value', None)
-        length = attributes.get('Length', {}).get('value', None)
-        precision = attributes.get('Precision', {}).get('value', None)
-        ordered_values = attributes.get('Ordered Values', {}).get('value', None)
-        sort_order = attributes.get('Sort Order', {}).get('value', None)
-        additional_properties = attributes.get('Additional Properties', {}).get('value', None)
-        effective_from = attributes.get('Effective From', {}).get('value', None)
-        effective_to = attributes.get('Effective To', {}).get('value', None)
-
-        position = attributes.get('Position', {}).get('value', None)
-        min_cardinality = attributes.get('Minimum Cardinality', {}).get('value', None)
-        max_cardinality = attributes.get('Maximum Cardinality', {}).get('value', None)
-        in_data_structure = attributes.get('In Data Structure', {}).get('value', None)
-        data_class = attributes.get('Data Class', {}).get('value', None)
-        glossary_term = attributes.get('Glossary Term', {}).get('value', None)
-        glossary_term_guid = attributes.get('Glossary Term', {}).get('guid', None)
-
-        # name_details_list = attributes.get("dict_list", None)
+        display_name = attributes.get('Display Name', {}).get('value', None)
+        qualified_name = attributes.get('Qualified Name',{}).get('value', None)
+        if qualified_name is None:
+            qualified_name = attributes.get('Display Name',{}).get('qualified_name', None)
+            if qualified_name is None:
+                logger.error(f"Element `{display_name}` does not have a valid specification? No qualified name..Review..\n\n___ ")
+                return None
+        prop_body = set_element_prop_body(object_type, qualified_name, attributes)
+        prop_body['namespace'] = attributes.get('Namespace', {}).get('value', None)
+        display_name = attributes.get('Display Name',{}).get('value', None)
 
         data_spec_name_list = attributes.get("In Data Specification", {}).get("name_list", "")
         data_spec_value = attributes.get("In Data Specification", {}).get("value", None)
@@ -736,79 +700,58 @@ def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str, d
         data_dict_name_list = attributes.get('In Data Dictionary', {}).get('name_list', "")
         data_dict_value_list = attributes.get('In Data Dictionary', {}).get('value', None)
         data_dict_guid_list = attributes.get("In Data Dictionary", {}).get("guid_list", None)
-
-        parent_data_field = attributes.get('Parent Data Field', {}).get('value', None)
-        parent_data_field_guid = attributes.get('Parent Data Field', {}).get('guid', None)
-
-        anchor_scope_guid = attributes.get('Anchor Scope GUID', {}).get('value', None)
-
-        collection_type = object_type
-        replace_all_props = True
-        if not valid:
-            if exists and object_action == "Create":
-                msg = (f"Create failed because Element `{display_name}` exists - changing `Create` to `Update` in "
-                       f"processed output \n\n___")
-                logger.error(msg)
-                return update_a_command(txt, object_action, object_type, qualified_name, guid)
-            else:
-                return None
-        elif object_action == "Update" and not exists:
-            logger.error(f"Element `{display_name}` does not exist! Updating result document with Create "
-                         f"object_action\n\n___")
-            return update_a_command(txt, object_action, object_type, qualified_name, guid)
-
-        else:
-            print(Markdown(f"==> Validation of {command} completed successfully! Proceeding to apply the changes.\n"))
+        merge_update = attributes.get('Merge Update', {}).get('value', True)
 
         try:
             if object_action == "Update":
-                body = {
-                    "class": "UpdateElementRequestBody", "externalSourceGUID": external_source_guid,
-                    "externalSourceName": external_source_name, "effectiveTime": effective_time,
-                    "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing, "properties": {
-                        "class": "DataStructureProperties", "qualifiedName": qualified_name,
-                        "displayName": display_name, "description": description, "namespace": namespace,
-                        "versionIdentifier": version_id, "additionalProperties": additional_properties,
-                        "effectiveFrom": effective_from, "effectiveTo": effective_to
-                        }
-                    }
-                egeria_client.update_data_structure_w_body(guid, body, replace_all_props)
-                logger.info(f"Updated element `{display_name}` with GUID {guid}")
-                core_props = egeria_client.get_data_structure_by_guid(guid, output_format='MD')
-
-                update_element_dictionary(qualified_name, {
-                    'guid': guid, 'display_name': display_name
+                if not exists:
+                    logger.error(f"Element `{qualified_name}` does not exist! Updating result document with Create "
+                                 f"object_action\n\n___")
+                    return update_a_command(txt, object_action, object_type, qualified_name, guid)
+                elif not valid:
+                    logger.error(f"Element `{qualified_name}` does not have a valid specification? Review..\n\n___ ")
+                    return None
+                else:
+                    update_body = set_update_body(object_type,attributes)
+                    update_body['properties'] = prop_body
+                    egeria_client.update_data_structure(guid, update_body)
+                    update_element_dictionary(qualified_name, {
+                        'guid': guid, 'display_name': display_name
+                        })
+                    logger.success(f"Updated  {object_type} `{display_name}` with GUID {guid}\n\n___")
+
+                    core_props = egeria_client.get_data_structure_by_guid(guid, output_format='MD')
+
+                    update_element_dictionary(qualified_name, {
+                        'guid': guid, 'display_name': display_name
                     })
 
-                update_data_collection_memberships(egeria_client, object_type, data_spec_guid_list, "DataSpec", guid,
-                                                   display_name, replace_all_props)
-                core_props += f"## In Data Dictionary\n\n{data_dict_name_list}\n\n"
-                core_props += f"## In Data Specification\n\n{data_spec_name_list}\n\n"
-                logger.success(f"Updated  {object_type} `{display_name}` with GUID {guid}\n\n___")
-                return core_props
+                    update_data_collection_memberships(egeria_client, object_type, data_spec_guid_list, "DataSpec", guid,
+                                                       display_name, merge_update)
+                    core_props += f"## In Data Dictionary\n\n{data_dict_name_list}\n\n"
+                    core_props += f"## In Data Specification\n\n{data_spec_name_list}\n\n"
+                    logger.success(f"Updated  {object_type} `{display_name}` with GUID {guid}\n\n___")
+                    return core_props
+
+
+            if not valid:
+                if exists and object_action == "Create":
+                    msg = (f"Create failed because Element `{display_name}` exists - changing `Create` to `Update` in "
+                           f"processed output \n\n___")
+                    logger.error(msg)
+                    return update_a_command(txt, object_action, object_type, qualified_name, guid)
+                else:
+                    return None
+
 
             elif object_action == "Create":
                 if exists:
                     logger.warning(f"\nTerm `{display_name}` already exists and result document updated\n\n___")
                     return update_a_command(txt, object_action, object_type, qualified_name, guid)
                 else:
-
-                    body = {
-                        "class": "NewElementRequestBody", "externalSourceGUID": external_source_guid,
-                        "externalSourceName": external_source_name, "effectiveTime": effective_time,
-                        "forLineage": False, "forDuplicateProcessing": False, "anchorGUID": anchor_guid,
-                        "isOwnAnchor": is_own_anchor, "parentGUID": parent_guid,
-                        "parentRelationshipTypeName": parent_relationship_type_name,
-                        "parentRelationshipProperties": parent_relationship_properties, "parentAtEnd1": parent_at_end1,
-                        "properties": {
-                            "class": "DataStructureProperties", "qualifiedName": qualified_name,
-                            "displayName": display_name, "description": description, "namespace": namespace,
-                            "versionIdentifier": version_id, "additionalProperties": additional_properties,
-                            "effectiveFrom": effective_from, "effectiveTo": effective_to
-                            }
-                        }
-
-                    guid = egeria_client.create_data_structure_w_body(body_slimmer(body))
+                    body = set_create_body(object_type, attributes)
+                    body['properties'] = prop_body
+                    guid = egeria_client.create_data_structure(body_slimmer(body))
                     if guid:
                         update_element_dictionary(qualified_name, {
                             'guid': guid, 'display_name': display_name
@@ -834,12 +777,11 @@ def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str, d
                         logger.error(f"Failed to create Data Structure `{display_name}`\n\n___")
                         return None
 
-
         except Exception as e:
             logger.error(f"Error performing {object_action}: {e}\n\n___")
             return None
-    else:
-        return None
+        else:
+            return None
 
 
 @logger.catch
@@ -884,38 +826,6 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
     elif directive == "process":
         logger.debug(json.dumps(parsed_output, indent=4))
 
-        external_source_guid = attributes.get('External Source Name', {}).get('guid', None)
-        external_source_name = attributes.get('External Source Name', {}).get('value', None)
-        effective_time = attributes.get('Effective Time', {}).get('value', None)
-        for_lineage = attributes.get('For Lineage', {}).get('value', None)
-        for_duplicate_processing = attributes.get('For Duplicate Processing', {}).get('value', None)
-        anchor_guid = attributes.get('Anchor ID', {}).get('guid', None)
-        is_own_anchor = attributes.get('Is Own Anchor', {}).get('value', None)
-        # parent_id = attributes.get('Parent ID', {}).get('value', None)
-        # parent_guid = attributes['Parent ID'].get('guid', None)
-        # parent_relationship_type_name = attributes.get('Parent Relationship Type Name', {}).get('value', None)
-        # parent_relationship_properties = attributes.get('Parent Relationship Properties',{}).get('value', None)
-        # parent_at_end1 = attributes.get('Parent at End1', {}).get('value', None)
-
-        namespace = attributes.get('Namespace', {}).get('value', None)
-        description = attributes.get('Description', {}).get('value', None)
-        version_id = attributes.get('Version Identifier', {}).get('value', None)
-        aliases = attributes.get('Aliases', {}).get('value', None)
-        name_patterns = attributes.get('Name Patterns', {}).get('value', None)
-        is_nullable = attributes.get('Is Nullable', {}).get('value', None)
-        default_value = attributes.get('Default Value', {}).get('value', None)
-        data_type = attributes.get('Data Type', {}).get('value', None)
-        min_length = attributes.get('Minimum Length', {}).get('value', None)
-        length = attributes.get('Length', {}).get('value', None)
-        precision = attributes.get('Precision', {}).get('value', None)
-        ordered_values = attributes.get('Ordered Values', {}).get('value', None)
-        sort_order = attributes.get('Sort Order', {}).get('value', None)
-        additional_properties = attributes.get('Additional Properties', {}).get('value', None)
-        effective_from = attributes.get('Effective From', {}).get('value', None)
-        effective_to = attributes.get('Effective To', {}).get('value', None)
-
-        glossary_term = attributes['Glossary Term'].get('value', None)
-        glossary_term_guid = attributes['Glossary Term'].get('guid', None)
 
         merge_update = attributes.get('Merge Update', {}).get('value', None)
 
@@ -929,13 +839,11 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
 
         data_class = attributes.get('Data Class', {}).get('value', None)
         data_class_guid = attributes.get('Data Class', {}).get('guid', None)
-
+        glossary_term = attributes.get('Glossary Term', {}).get('value', None)
         glossary_term_guid = attributes.get('Glossary Term', {}).get('guid', None)
         if glossary_term_guid:
             glossary_term_guid = [glossary_term_guid]
 
-        glossary_term_guid = attributes.get('Glossary Term', {}).get('guid', None)
-
         # name_details_list = attributes.get("dict_list", None)
 
         in_data_spec = attributes.get("In Data Specification", {}).get("value", None)  # this is a [dict]
@@ -967,6 +875,8 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
         else:
             print(Markdown(f"==> Validation of {command} completed successfully! Proceeding to apply the changes.\n"))
 
+        props_body = set_data_field_body(object_type, qualified_name, attributes)
+
         try:
             if object_action == "Update":
                 if not exists:
@@ -975,29 +885,15 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                     return update_a_command(txt, object_action, object_type, qualified_name, guid)
 
                 # first update the base data field
-                body = {
-                    "class": "UpdateElementRequestBody", "externalSourceGUID": external_source_guid,
-                    "externalSourceName": external_source_name, "effectiveTime": effective_time,
-                    "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing, "properties": {
-                        "class": "DataFieldProperties", "qualifiedName": qualified_name, "displayName": display_name,
-                        "namespace": namespace, "description": description, "versionIdentifier": version_id,
-                        "aliases": aliases, "namePatterns": name_patterns, "isDeprecated": False,
-                        "isNullable": is_nullable, "defaultValue": default_value, "dataType": data_type,
-                        "minimumLength": min_length, "length": length, "precision": precision,
-                        "orderedValues": ordered_values, "sortOrder": sort_order,
-                        "additionalProperties": additional_properties, "effectiveFrom": effective_from,
-                        "effectiveTo": effective_to
-                        }
-                    }
-
-                egeria_client.update_data_field(guid, body, not merge_update)
+                body = set_update_body(object_type, attributes)
+                body['properties'] = props_body
+                egeria_client.update_data_field(guid, body)
                 logger.success(f"Updated  {object_type} `{display_name}` with GUID {guid}")
                 # Update data dictionary membership
                 update_element_dictionary(qualified_name, {
                     'guid': guid, 'display_name': display_name
                     })
-                core_props = egeria_client.find_data_fields(qualified_name,
-                                                            output_format='MD')  ## update back to by_guid?
+                core_props = egeria_client.get_data_field_by_guid(guid, output_format='MD')  ## update back to by_guid?
 
                 # existing_data_field = egeria_client.get_data_field_by_guid(guid, output_format='JSON')
 
@@ -1028,17 +924,10 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                     return update_a_command(txt, object_action, object_type, qualified_name, guid)
                 else:
                     # First lets create the data field
-                    body = {
-                        "class": "NewElementRequestBody", "properties": {
-                            "class": "DataFieldProperties", "qualifiedName": qualified_name,
-                            "displayName": display_name, "namespace": namespace, "description": description,
-                            "versionIdentifier": version_id, "aliases": aliases, "namePatterns": name_patterns,
-                            "isDeprecated": False, "isNullable": is_nullable, "defaultValue": default_value,
-                            "dataType": data_type, "minimumLength": min_length, "length": length,
-                            "precision": precision, "orderedValues": ordered_values, "sortOrder": sort_order,
-                            "additionalProperties": additional_properties
-                            }
-                        }
+                    body = set_create_body(object_type, attributes)
+                    body['properties'] = props_body
+                    parent_guid = body.get('parentGuid', None)
+
                     guid = egeria_client.create_data_field(body)
                     if guid:
                         # Now update our element dictionary with the new information
@@ -1046,7 +935,7 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                             'guid': guid, 'display_name': display_name
                             })
                         # Start assembling the information we will present back out
-                        core_props = egeria_client.get_data_field_by_guid(guid, None, 'MD')
+                        core_props = egeria_client.get_data_field_by_guid(guid, 'MD')
 
                         # Add the field to any data dictionaries
                         if in_data_dictionary:
@@ -1059,27 +948,27 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                             core_props += f"\n\n## In Data Structure\n\n{in_data_structure_names}\n\n"
                             for ds_guid in data_structure_guid_list:
                                 # todo This is too naive? - need to better accommodate the relationship
-                                df_body = {
-                                    "class": "RelationshipRequestBody", "properties": {
-                                        "class": "MemberDataFieldProperties", "dataFieldPosition": position,
-                                        "minCardinality": min_cardinality, "maxCardinality": max_cardinality,
-                                        }
-                                    }
+                                prop_body = set_rel_prop_body("MemberDataFieldProperties", attributes)
+                                prop_body["position"] = position
+                                prop_body["minimumCardinality"] = min_cardinality
+                                prop_body["maximumCardinality"] = max_cardinality
+
+                                body = set_rel_request_body(object_type, attributes)
+                                body['properties'] = prop_body
 
                                 msg = f"Adding field to structure {ds_guid}"
                                 logger.info(msg)
-                                egeria_client.link_member_data_field(ds_guid, guid, df_body)
+                                egeria_client.link_member_data_field(ds_guid, guid, body)
                             core_props += f"\n\n## In Data Structure {in_data_structure_names}\n\n"
 
                         if glossary_term:
                             if glossary_term_guid:
-                                glossary_body = {
-                                    "class": "RelationshipRequestBody", "externalSourceGUID": external_source_guid,
-                                    "externalSourceName": external_source_name, "effectiveTime": effective_time,
-                                    "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing
-                                    }
+                                term_body = set_rel_prop_body("GlossaryTermProperties", attributes)
+                                body = set_rel_request_body(object_type, attributes)
+                                body['properties'] = term_body
+
                                 core_props += f"\n\n## Glossary Term \n\n{glossary_term}\n\n"
-                                egeria_client.link_semantic_definition(guid, glossary_term_guid, glossary_body)
+                                egeria_client.link_semantic_definition(guid, glossary_term_guid, term_body)
 
                         if parent_data_field_guids:
                             # parent_df_body = {
@@ -1095,11 +984,11 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
 
                         # Link data class
                         if data_class:
-                            body = {
-                                "class": "RelationshipRequestBody", "externalSourceGUID": external_source_guid,
-                                "externalSourceName": external_source_name, "effectiveTime": effective_time,
-                                "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing
-                                }
+                            prop_body = set_rel_prop_body("DataClassProperties", attributes)
+                            body = set_rel_request_body(object_type, attributes)
+                            body['properties'] = prop_body
+
+                            core_props += f"\n\n## Data Class\n\n{data_class}\n\n"
                             egeria_client.link_data_class_definition(guid, data_class_guid, body)
                             msg = f"Adding data class `{data_class}` to data field {display_name}"
                             logger.info(msg)
@@ -1372,292 +1261,3 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
     else:
         return None
 
-
-@logger.catch
-def process_data_collection_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[
-    str]:
-    """
-    Processes a Data Dictionary list object_action by extracting key attributes such as
-     search string from the given text.
-
-    :param txt: A string representing the input cell to be processed for
-        extracting term-related attributes.
-    :param directive: an optional string indicating the directive to be used - display, validate or execute
-    :return: A string summarizing the outcome of the processing.
-    """
-    command, object_type, object_action = extract_command_plus(txt)
-    print(Markdown(f"# {command}\n"))
-    if object_type in ["Data Dictionary", "Data Dictionaries", "DataDict", "DataDictionary"]:
-        col_type = "DataDictionary"
-    elif object_type in ["Data Specification", "Data Specifications", "Data Specs"]:
-        col_type = "DataSpec"
-    else:
-        col_type = "Collection"
-
-    parsed_output = parse_view_command(egeria_client, object_type, object_action, txt, directive)
-
-
-
-    valid = parsed_output['valid']
-    print(Markdown(f"Performing {command}"))
-    print(Markdown(parsed_output['display']))
-
-    attr = parsed_output.get('attributes',{})
-    effective_time = attr.get('effectiveTime', {}).get('value', None)
-    as_of_time = attr.get('asOfTime', {}).get('value', None)
-    for_duplicate_processing = attr.get('forDuplicateProcessing', {}).get('value', False)
-    for_lineage = attr.get('forLineage',{}).get('value', False)
-    limit_result_by_status = attr.get('limitResultsByStatus',{}).get('value', ['ACTIVE'])
-    sequencing_property = attr.get('sequencingProperty',{}).get('value',"qualifiedName" )
-    sequencing_order = attr.get('sequencingOrder',{}).get('value', "PROPERTY_ASCENDING")
-    search_string = attr.get('Search String', {}).get('value', '*')
-    output_format = attr.get('Output Format', {}).get('value', 'LIST')
-    detailed = attr.get('Detailed', {}).get('value', False)
-
-    if directive == "display":
-        return None
-    elif directive == "validate":
-        if valid:
-            print(Markdown(f"==> Validation of {command} completed successfully!\n"))
-        else:
-            msg = f"Validation failed for object_action `{command}`\n"
-            logger.error(msg)
-        return valid
-
-    elif directive == "process":
-        try:
-            if not valid:  # First validate the command before we process it
-                msg = f"Validation failed for {object_action} `{object_type}`\n"
-                logger.error(msg)
-                return None
-
-            list_md = f"\n# `{col_type}` with filter: `{search_string}`\n\n"
-            body = {
-                    "class": "FilterRequestBody",
-                    "asOfTime": as_of_time,
-                    "effectiveTime": effective_time,
-                    "forLineage": for_lineage,
-                    "forDuplicateProcessing": for_duplicate_processing,
-                    "limitResultsByStatus": limit_result_by_status,
-                    "sequencingOrder": sequencing_order,
-                    "sequencingProperty": sequencing_property,
-                    "filter": search_string,
-                }
-
-            struct = egeria_client.find_collections_w_body(body, col_type, output_format=output_format)
-            if output_format == "DICT":
-                list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
-            else:
-                list_md += struct
-            logger.info(f"Wrote `{col_type}` for search string: `{search_string}`")
-
-            return list_md
-
-        except Exception as e:
-            logger.error(f"Error performing {command}: {e}")
-            console.print_exception(show_locals=True)
-            return None
-    else:
-        return None
-
-
-def process_data_structure_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[
-    str]:
-    """
-    Processes a Data Dictionary list object_action by extracting key attributes such as
-     search string from the given text.
-
-    :param txt: A string representing the input cell to be processed for
-        extracting term-related attributes.
-    :param directive: an optional string indicating the directive to be used - display, validate or execute
-    :return: A string summarizing the outcome of the processing.
-    """
-    command, object_type, object_action = extract_command_plus(txt)
-    print(Markdown(f"# {command}\n"))
-
-    parsed_output = parse_view_command(egeria_client, object_type, object_action, txt, directive)
-
-    attributes = parsed_output['attributes']
-
-    valid = parsed_output['valid']
-    print(Markdown(f"Performing {command}"))
-    print(Markdown(parsed_output['display']))
-
-    if directive == "display":
-        return None
-    elif directive == "validate":
-        if valid:
-            print(Markdown(f"==> Validation of {command} completed successfully!\n"))
-        else:
-            msg = f"Validation failed for object_action `{command}`\n"
-            logger.error(msg)
-        return valid
-
-    elif directive == "process":
-        attributes = parsed_output['attributes']
-        search_string = attributes.get('Search String', {}).get('value', '*')
-        output_format = attributes.get('Output Format', {}).get('value', 'LIST')
-        detailed = attributes.get('Detailed', {}).get('value', False)
-
-        try:
-            if not valid:  # First validate the command before we process it
-                msg = f"Validation failed for {object_action} `{object_type}`\n"
-                logger.error(msg)
-                return None
-
-            list_md = f"\n# `{object_type}` with filter: `{search_string}`\n\n"
-            struct = egeria_client.find_data_structures(search_string, output_format=output_format)
-
-            if output_format == "DICT":
-                list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
-            else:
-                list_md += struct
-            logger.info(f"Wrote `{object_type}` for search string: `{search_string}`")
-
-            return list_md
-
-        except Exception as e:
-            logger.error(f"Error performing {command}: {e}")
-            console.print_exception(show_locals=True)
-            return None
-    else:
-        return None
-
-
-def process_data_field_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[str]:
-    """
-    Processes a Data Dictionary list object_action by extracting key attributes such as
-     search string from the given text.
-
-    :param txt: A string representing the input cell to be processed for
-        extracting term-related attributes.
-    :param directive: an optional string indicating the directive to be used - display, validate or execute
-    :return: A string summarizing the outcome of the processing.
-    """
-    command, object_type, object_action = extract_command_plus(txt)
-    print(Markdown(f"# {command}\n"))
-
-    parsed_output = parse_view_command(egeria_client, object_type, object_action, txt, directive)
-
-    attributes = parsed_output['attributes']
-
-    valid = parsed_output['valid']
-    print(Markdown(f"Performing {command}"))
-    print(Markdown(parsed_output['display']))
-
-    if directive == "display":
-        return None
-    elif directive == "validate":
-        if valid:
-            print(Markdown(f"==> Validation of {command} completed successfully!\n"))
-        else:
-            msg = f"Validation failed for object_action `{command}`\n"
-            logger.error(msg)
-        return valid
-
-    elif directive == "process":
-        attributes = parsed_output['attributes']
-        search_string = attributes.get('Search String', {}).get('value', '*')
-        output_format = attributes.get('Output Format', {}).get('value', 'LIST')
-        detailed = attributes.get('Detailed', {}).get('value', False)
-        as_of_time = attributes.get('AsOfTime', {}).get('value', None)
-        effective_time = attributes.get('Effective Time', {}).get('value', None)
-        sort_order = attributes.get('Sort Order', {}).get('value', None)
-        order_property = attributes.get('Order Property', {}).get('value', None)
-        starts_with = attributes.get('Start With', {}).get('value', True)
-        ends_with = attributes.get('End With', {}).get('value', False)
-        ignore_case = attributes.get('Ignore Case', {}).get('value', False)
-        start_from = attributes.get('Start From', {}).get('value', 0)
-        page_size = attributes.get('Page Size', {}).get('value', None)
-
-        try:
-            if not valid:  # First validate the command before we process it
-                msg = f"Validation failed for {object_action} `{object_type}`\n"
-                logger.error(msg)
-                return None
-
-            list_md = f"\n# `{object_type}` with filter: `{search_string}`\n\n"
-            body = {
-                "class": "FilterRequestBody", "asOfTime": as_of_time, "effectiveTime": effective_time,
-                "forLineage": False, "forDuplicateProcessing": False, "limitResultsByStatus": ["ACTIVE"],
-                "sequencingOrder": sort_order, "sequencingProperty": order_property, "filter": search_string,
-                }
-            struct = egeria_client.find_data_fields_w_body(body, start_from, page_size, starts_with, ends_with,
-                                                           ignore_case, output_format)
-
-            if output_format == "DICT":
-                list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
-            else:
-                list_md += struct
-            logger.info(f"Wrote `{object_type}` for search string: `{search_string}`")
-
-            return list_md
-
-        except Exception as e:
-            logger.error(f"Error performing {command}: {e}")
-            console.print_exception(show_locals=True)
-            return None
-    else:
-        return None
-
-
-def process_data_class_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[str]:
-    """
-    Processes a Data Dictionary list object_action by extracting key attributes such as
-     search string from the given text.
-
-    :param txt: A string representing the input cell to be processed for
-        extracting term-related attributes.
-    :param directive: an optional string indicating the directive to be used - display, validate or execute
-    :return: A string summarizing the outcome of the processing.
-    """
-    command, object_type, object_action = extract_command_plus(txt)
-    print(Markdown(f"# {command}\n"))
-
-    parsed_output = parse_view_command(egeria_client, object_type, object_action, txt, directive)
-
-    attributes = parsed_output['attributes']
-
-    valid = parsed_output['valid']
-    print(Markdown(f"Performing {command}"))
-    print(Markdown(parsed_output['display']))
-
-    if directive == "display":
-        return None
-    elif directive == "validate":
-        if valid:
-            print(Markdown(f"==> Validation of {command} completed successfully!\n"))
-        else:
-            msg = f"Validation failed for object_action `{command}`\n"
-            logger.error(msg)
-        return valid
-
-    elif directive == "process":
-        attributes = parsed_output['attributes']
-        search_string = attributes.get('Search String', {}).get('value', '*')
-        output_format = attributes.get('Output Format', {}).get('value', 'LIST')
-        detailed = attributes.get('Detailed', {}).get('value', False)
-
-        try:
-            if not valid:  # First validate the command before we process it
-                msg = f"Validation failed for {object_action} `{object_type}`\n"
-                logger.error(msg)
-                return None
-
-            list_md = f"\n# `{object_type}` with filter: `{search_string}`\n\n"
-            struct = egeria_client.find_data_classes(search_string, output_format=output_format)
-
-            if output_format == "DICT":
-                list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
-            else:
-                list_md += struct
-            logger.info(f"Wrote `{object_type}` for search string: `{search_string}`")
-
-            return list_md
-
-        except Exception as e:
-            logger.error(f"Error performing {command}: {e}")
-            console.print_exception(show_locals=True)
-            return None
-    else:
-        return None