pyegeria 5.4.0.dev3__py3-none-any.whl → 5.4.0.dev6__py3-none-any.whl

md_processing/md_commands/data_designer_commands.py

@@ -2,7 +2,7 @@
 This file contains term-related object_action functions for processing Egeria Markdown
 """
 import json
-import sys
+import sys, os
 from typing import Optional
 
 from loguru import logger
@@ -17,6 +17,26 @@ from md_processing.md_processing_utils.md_processing_constants import (load_comm
 from pyegeria import DEBUG_LEVEL, body_slimmer
 from pyegeria.egeria_tech_client import EgeriaTech
 
+
+GERIA_METADATA_STORE = os.environ.get("EGERIA_METADATA_STORE", "active-metadata-store")
+EGERIA_KAFKA_ENDPOINT = os.environ.get("KAFKA_ENDPOINT", "localhost:9092")
+EGERIA_PLATFORM_URL = os.environ.get("EGERIA_PLATFORM_URL", "https://localhost:9443")
+EGERIA_VIEW_SERVER = os.environ.get("EGERIA_VIEW_SERVER", "view-server")
+EGERIA_VIEW_SERVER_URL = os.environ.get("EGERIA_VIEW_SERVER_URL", "https://localhost:9443")
+EGERIA_INTEGRATION_DAEMON = os.environ.get("EGERIA_INTEGRATION_DAEMON", "integration-daemon")
+EGERIA_INTEGRATION_DAEMON_URL = os.environ.get("EGERIA_INTEGRATION_DAEMON_URL", "https://localhost:9443")
+EGERIA_ADMIN_USER = os.environ.get("ADMIN_USER", "garygeeke")
+EGERIA_ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "secret")
+EGERIA_USER = os.environ.get("EGERIA_USER", "erinoverview")
+EGERIA_USER_PASSWORD = os.environ.get("EGERIA_USER_PASSWORD", "secret")
+EGERIA_WIDTH = os.environ.get("EGERIA_WIDTH", 220)
+EGERIA_JUPYTER = os.environ.get("EGERIA_JUPYTER", False)
+EGERIA_HOME_GLOSSARY_GUID = os.environ.get("EGERIA_HOME_GLOSSARY_GUID", None)
+EGERIA_GLOSSARY_PATH = os.environ.get("EGERIA_GLOSSARY_PATH", None)
+EGERIA_ROOT_PATH = os.environ.get("EGERIA_ROOT_PATH", "../../")
+EGERIA_INBOX_PATH = os.environ.get("EGERIA_INBOX_PATH", "md_processing/dr_egeria_inbox")
+EGERIA_OUTBOX_PATH = os.environ.get("EGERIA_OUTBOX_PATH", "md_processing/dr_egeria_outbox")
+
 load_commands('commands.json')
 debug_level = DEBUG_LEVEL
 
@@ -24,8 +44,11 @@ console = Console(width=int(200))
 
 log_format = "D {time} | {level} | {function} | {line} | {message} | {extra}"
 logger.remove()
-logger.add(sys.stderr, level="INFO", format=log_format, colorize=True)
-logger.add("debug_log.log", rotation="1 day", retention="1 week", compression="zip", level="TRACE", format=log_format,
+logger.add(sys.stderr, level="ERROR", format=log_format, colorize=True)
+full_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_INBOX_PATH, "data_designer_debug.log")
+# logger.add(full_file_path, rotation="1 day", retention="1 week", compression="zip", level="TRACE", format=log_format,
+#            colorize=True)
+logger.add("debug_log", rotation="1 day", retention="1 week", compression="zip", level="TRACE", format=log_format,
            colorize=True)
 
 
@@ -43,10 +66,13 @@ def add_member_to_data_collections(egeria_client: EgeriaTech, collection_list: l
         "notes": "Added by Dr.Egeria"
         }
     try:
-        for collection in collection_list:
-            egeria_client.add_to_collection(collection, guid, body)
-            msg = f"Added `{display_name}` member to `{collection}`"
-            logger.info(msg)
+        if collection_list is not None:
+            for collection in collection_list:
+                egeria_client.add_to_collection(collection, guid, body)
+                msg = f"Added `{display_name}` member to `{collection}`"
+                logger.info(msg)
+        else:
+            logger.info("There were no data collections to add.")
         return
 
     except Exception as e:
@@ -90,8 +116,8 @@ def update_data_collection_memberships(egeria_client: EgeriaTech, entity_type: s
 
         coll_list = egeria_client.get_data_memberships(get_command, guid)
         if coll_list is None:
-            logger.error("Internal Error -> the collection list cannot be None")
-            return
+            logger.warning("Unexpected -> the collection list was None - assigning empty dict")
+            coll_list = {}
         # compare the existing collections to desired collections
         if collection_class == "DataDictionary":
             as_is = set(coll_list.get("DictList", {}))
@@ -163,9 +189,17 @@ def sync_data_field_rel_elements(egeria_client: EgeriaTech, structure_list: list
     TODO: Need to add data class support when ready and may need to revisit bodies.
 
     """
+    if terms:
+        terms = [terms]
+
     if replace_all_props:
         rel_el_list = egeria_client.get_data_field_rel_elements(guid)
         # should I throw an exception if empty?
+        if rel_el_list is None:
+            logger.warning("Unexpected -> the list was None - assigning empty list")
+            rel_el_list = {}
+
+
         as_is_data_structs = set(rel_el_list.get("data_structure_guids", []))
         as_is_parent_fields = set(rel_el_list.get("parent_guids", []))
         as_is_assigned_meanings = set(rel_el_list.get("assigned_meanings_guids", []))
@@ -174,7 +208,7 @@ def sync_data_field_rel_elements(egeria_client: EgeriaTech, structure_list: list
         to_be_data_structs = set(structure_list) if structure_list is not None else set()
         to_be_parent_fields = set(parent_field_list) if parent_field_list is not None else set()
         to_be_assigned_meanings = set(terms) if terms is not None else set()
-        to_be_data_classes = set(data_class_guid) if data_class_guid is not None else set()
+        to_be_data_classes = set([data_class_guid]) if data_class_guid is not None else set()
 
         logger.trace(f"as_is_data_structs: {list(as_is_data_structs)} to_be_data_struct: {list(to_be_data_structs)}")
         logger.trace(
@@ -216,7 +250,7 @@ def sync_data_field_rel_elements(egeria_client: EgeriaTech, structure_list: list
 
         terms_to_remove = as_is_assigned_meanings - to_be_assigned_meanings
         logger.trace(f"terms_to_remove: {list(terms_to_remove)}")
-        if len(terms_to_remove) > 0:
+        if terms:
             for term in terms_to_remove:
                 egeria_client.detach_semantic_definition(guid, term, None)
                 msg = f"Removed `{term}` from `{display_name}`"
@@ -233,94 +267,160 @@ def sync_data_field_rel_elements(egeria_client: EgeriaTech, structure_list: list
         logger.trace(f"classes_to_remove: {list(classes_to_remove)}")
         if len(terms_to_remove) > 0:
             for dc in classes_to_remove:
-                egeria_client.detach_data_class_definition(guid, dc)
+                body = {
+                    "class": "MetadataSourceRequestBody",
+                    "forLineage": False,
+                    "forDuplicateProcessing": False
+                }
+                egeria_client.detach_data_class_definition(guid, dc, body)
                 msg = f"Removed `{dc}` from `{display_name}`"
                 logger.trace(msg)
         classes_to_add = to_be_data_classes - as_is_data_classes
         logger.trace(f"classes_to_add: {list(classes_to_add)}")
         if len(terms_to_add) > 0:
             for dc in classes_to_add:
-                egeria_client.link_data_class_definition(guid, dc)
+                body = {
+                    "class": "RelationshipRequestBody",
+                    "forLineage": False,
+                    "forDuplicateProcessing": False
+                    }
+                egeria_client.link_data_class_definition(guid, dc, body)
                 msg = f"Added `{dc}` to`{display_name}`"
                 logger.trace(msg)
 
 
     else:  # merge - add field to related elements
-        add_field_to_data_structures(egeria_client, display_name, structure_list, guid)
-        msg = f"Added `{display_name}` to `{structure_list}`"
-        logger.trace(msg)
-
-        for field in parent_field_list:
-            egeria_client.link_nested_data_field(field, guid, None)
-            msg = f"Added `{display_name}` to `{field}`"
-            logger.trace(msg)
-        for term in terms:
-            egeria_client.link_semantic_definition(guid, term, None)
-            msg = f"Added `{term}` to `{display_name}`"
+        if structure_list:
+            add_field_to_data_structures(egeria_client, display_name, structure_list, guid)
+            msg = f"Added `{display_name}` to `{structure_list}`"
             logger.trace(msg)
 
-        egeria_client.link_data_class_definition(guid, data_class_guid)
-        msg = f"Added `{data_class_guid}` to `{display_name}`"
-        logger.trace(msg)
+        if parent_field_list:
+            for field in parent_field_list:
+                egeria_client.link_nested_data_field(field, guid, None)
+                msg = f"Added `{display_name}` to `{field}`"
+                logger.trace(msg)
+        if terms:
+            for term in terms:
+                egeria_client.link_semantic_definition(guid, term, None)
+                msg = f"Added `{term}` to `{display_name}`"
+                logger.trace(msg)
+
+        if data_class_guid:
+            egeria_client.link_data_class_definition(guid, data_class_guid)
+            msg = f"Added `{data_class_guid}` to `{display_name}`"
+            logger.trace(msg)
 
 
 @logger.catch
 def sync_data_class_rel_elements(egeria_client: EgeriaTech, containing_data_class_guids: list, terms: list,
-                                 containing_data_classes: list, super_data_classes: list, guid: str, display_name: str,
+                                 specializes_data_classes: list, guid: str, display_name: str,
                                  replace_all_props: bool = True) -> None:
     """Sync a data class' related elements.
 
-    TODO:  may need to revisit bodies.
-
     """
+    if terms:
+        terms = [terms]
+
     if replace_all_props:
-        rel_el_list = egeria_client.get_data_field_rel_elements(guid)
-        # should I throw an exception if empty?
-        as_is_containing_data_class_guids = set(rel_el_list.get("data_structure_guids", []))
-        as_is_parent_fields = set(rel_el_list.get("parent_guids", []))
+        rel_el_list = egeria_client.get_data_class_rel_elements(guid)
+        if rel_el_list is None:
+            logger.warning("Unexpected -> the list was None - assigning empty list")
+            rel_el_list = {}
+        if terms:
+            terms = [terms]
+
+        as_is_nested_classes = set(rel_el_list.get("nested_data_class_guids", []))
         as_is_assigned_meanings = set(rel_el_list.get("assigned_meanings_guids", []))
-        as_is_data_classes = set(rel_el_list.get("data_class_guids", []))
+        as_is_specialized_classes = set(rel_el_list.get("specialized_data_class_guids", []))
+
+        to_be_nested_classes = set(containing_data_class_guids) if containing_data_class_guids is not None else set()
+        to_be_assigned_meanings = set(terms) if terms is not None else set()
+        to_be_specialized_classes = set([specializes_data_classes]) if specializes_data_classes is not None else set()
+
+        logger.trace(
+            f"as_is_nested_classes: {list(as_is_nested_classes)} to_be_nested_classes: {list(to_be_nested_classes)}")
+        logger.trace(f"as_is_assigned_meanings: {list(as_is_assigned_meanings)} to_be_assigned_meanings: "
+                     f"{list(to_be_assigned_meanings)}")
+        logger.trace(f"as_is_specialized_classes: {list(as_is_specialized_classes)} to_be_specizialized_data_classes: "
+                     f"{list(to_be_specialized_classes)}")
+
+
+        nested_classes_to_remove = to_be_nested_classes - as_is_nested_classes
+        logger.trace(f"nested_classes_to_remove: {list(nested_classes_to_remove)}")
+        if len(nested_classes_to_remove) > 0:
+            for field in nested_classes_to_remove:
+                egeria_client.detach_nested_data_class(field, guid, None)
+                msg = f"Removed `{display_name}` from field `{field}`"
+                logger.trace(msg)
+        nested_classes_to_add = to_be_nested_classes - as_is_nested_classes
+        logger.trace(f"nested_classes_to_add: {list(nested_classes_to_add)}")
+        if len(nested_classes_to_add) > 0:
+            for field in nested_classes_to_add:
+                egeria_client.link_nested_data_class(field, guid, None)
+                msg = f"Added `{display_name}` to field `{field}`"
+                logger.trace(msg)
+
+        terms_to_remove = as_is_assigned_meanings - to_be_assigned_meanings
+        logger.trace(f"terms_to_remove: {list(terms_to_remove)}")
+        if len(terms_to_remove) > 0:
+            for term in terms_to_remove:
+                egeria_client.detach_semantic_definition(guid, term, None)
+                msg = f"Removed `{term}` from `{display_name}`"
+                logger.trace(msg)
+        terms_to_add = to_be_assigned_meanings - as_is_assigned_meanings
+        logger.trace(f"terms_to_add: {list(terms_to_add)}")
+        if len(terms_to_add) > 0:
+            for term in terms_to_add:
+                egeria_client.link_semantic_definition(guid, term, None)
+                msg = f"Added `{term}` to`{display_name}`"
+                logger.trace(msg)
+
+        specialized_classes_to_remove = as_is_specialized_classes - to_be_specialized_classes
+        logger.trace(f"classes_to_remove: {list(specialized_classes_to_remove)}")
+        if len(terms_to_remove) > 0:
+            for dc in specialized_classes_to_remove:
+                body = {
+                    "class": "MetadataSourceRequestBody",
+                    "forLineage": False,
+                    "forDuplicateProcessing": False
+                    }
+                egeria_client.detach_specialist_data_class(guid, dc, body)
+                msg = f"Removed `{dc}` from `{display_name}`"
+                logger.trace(msg)
+        specialized_classes_to_add = to_be_specialized_classes - as_is_specialized_classes
+        logger.trace(f"classes_to_add: {list(specialized_classes_to_add)}")
+        if len(specialized_classes_to_add) > 0:
+            for dc in specialized_classes_to_add:
+                body = {
+                    "class": "RelationshipRequestBody",
+                    "forLineage": False,
+                    "forDuplicateProcessing": False
+                    }
+                egeria_client.link_specialist_data_class(guid, dc, body)
+                msg = f"Added `{dc}` to`{display_name}`"
+                logger.trace(msg)
+
+
+    else:  # merge - add field to related elements
+        if containing_data_class_guids:
+            for field in containing_data_class_guids:
+                egeria_client.link_nested_data_class(field, guid, None)
+                msg = f"Added `{display_name}` to `{field}`"
+                logger.trace(msg)
+
+        if terms:
+            for term in terms:
+                egeria_client.link_semantic_definition(guid, term, None)
+                msg = f"Added `{term}` to `{display_name}`"
+                logger.trace(msg)
+        if specializes_data_classes:
+            for el in specializes_data_classes:
+                egeria_client.link_specialist_data_class(guid, el)
+            msg = f"Linked `{el}` to `{display_name}`"
+            logger.trace(msg)
+
 
-        # to_be_data_structs = set(structure_list) if structure_list is not None else set()  #
-        # to_be_parent_fields = set(parent_field_list) if parent_field_list is not None else set()  #
-        # to_be_assigned_meanings = set(terms) if data_class_list is not None else set()  #
-        # to_be_assigned_data_classes = set(data_class_list) if data_class_list is not None else set()  #  #
-        # logger.trace(f"as_is_data_structs: {list(as_is_data_structs)} to_be_data_struct: {list(
-        # to_be_data_structs)}")  #     logger.trace(  #         f"as_is_parent_fields: {list(as_is_parent_fields)}
-        # to_be_parent_fields: {list(to_be_parent_fields)}")  #     logger.trace(  #
-        # f"as_is_assigned_meanings: {list(as_is_assigned_meanings)} to_be_assigned_meanings: "  #         f"{list(
-        # to_be_assigned_meanings)}")  #     logger.trace(  #         f"as_is_data_classes: {list(
-        # as_is_data_classes)} to_be_assigned_data_classes: "  #         f"{list(to_be_assigned_data_classes)}")  #
-        # data_struct_to_remove = as_is_data_structs - to_be_data_structs  #     logger.trace(
-        # f"data_struct_to_remove: {list(data_struct_to_remove)}")  #     if len(data_struct_to_remove) > 0:  #
-        # for ds in data_struct_to_remove:  #             egeria_client.detach_member_data_field(ds, guid,
-        # None)  #             msg = f"Removed `{display_name}` from structure `{ds}`"  #             logger.trace(
-        # msg)  #     data_struct_to_add = to_be_data_structs - as_is_data_structs  #     logger.trace(
-        # f"data_struct_to_add: {list(data_struct_to_add)}")  #     if len(data_struct_to_add) > 0:  #         for ds
-        # in data_struct_to_add:  #             egeria_client.link_member_data_field(ds, guid, None)  #
-        # msg = f"Added `{display_name}` to structure `{ds}`"  #             logger.trace(msg)  #  #
-        # parent_field_to_remove = to_be_parent_fields - as_is_parent_fields  #     logger.trace(
-        # f"parent_field_to_remove: {list(parent_field_to_remove)}")  #     if len(parent_field_to_remove) > 0:  #
-        # for field in parent_field_to_remove:  #             egeria_client.detach_nested_data_field(field, guid,
-        # None)  #             msg = f"Removed `{display_name}` from field `{field}`"  #             logger.trace(
-        # msg)  #     parent_field_to_add = to_be_parent_fields - as_is_parent_fields  #     logger.trace(
-        # f"parent_field_to_add: {list(parent_field_to_add)}")  #     if len(parent_field_to_add) > 0:  #         for
-        # field in parent_field_to_add:  #             egeria_client.link_nested_data_field(field, guid, None)  #
-        # msg = f"Added `{display_name}` to field `{field}`"  #             logger.trace(msg)  #  #
-        # terms_to_remove = as_is_assigned_meanings - to_be_assigned_meanings  #     logger.trace(f"terms_to_remove:
-        # {list(terms_to_remove)}")  #     if len(terms_to_remove) > 0:  #         for term in terms_to_remove:  #
-        # egeria_client.detach_semantic_definition(guid, term, None)  #             msg = f"Removed `{term}` from `{
-        # display_name}`"  #             logger.trace(msg)  #     terms_to_add = to_be_assigned_meanings -
-        # as_is_assigned_meanings  #     logger.trace(f"terms_to_add: {list(terms_to_add)}")  #     if len(
-        # terms_to_add) > 0:  #         for term in terms_to_add:  #
-        # egeria_client.link_semantic_definition(guid, term, None)  #             msg = f"Added `{term}` to`{
-        # display_name}`"  #             logger.trace(msg)  #  # else:  # merge - add field to related elements  #
-        # add_field_to_data_structures(egeria_client, display_name, structure_list, guid)  #     msg = f"Added `{
-        # display_name}` to `{structure_list}`"  #     logger.trace(msg)  #  #     for field in parent_field_list:  #
-        # egeria_client.link_nested_data_field(field, guid, None)  #         msg = f"Added `{display_name}` to `{
-        # field}`"  #         logger.trace(msg)  #     for term in terms:  #
-        # egeria_client.link_semantic_definition(guid, term, None)  #         msg = f"Added `{term}` to `{
-        # display_name}`"  #         logger.trace(msg)
 
 
 @logger.catch
@@ -683,7 +783,7 @@ def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str, d
         try:
             if object_action == "Update":
                 body = {
-                    "class": "UpdateDataStructureRequestBody", "externalSourceGUID": external_source_guid,
+                    "class": "UpdateElementRequestBody", "externalSourceGUID": external_source_guid,
                     "externalSourceName": external_source_name, "effectiveTime": effective_time,
                     "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing, "properties": {
                         "class": "DataStructureProperties", "qualifiedName": qualified_name,
@@ -714,7 +814,7 @@ def process_data_structure_upsert_command(egeria_client: EgeriaTech, txt: str, d
                 else:
 
                     body = {
-                        "class": "NewDataStructureRequestBody", "externalSourceGUID": external_source_guid,
+                        "class": "NewElementRequestBody", "externalSourceGUID": external_source_guid,
                         "externalSourceName": external_source_name, "effectiveTime": effective_time,
                         "forLineage": False, "forDuplicateProcessing": False, "anchorGUID": anchor_guid,
                         "isOwnAnchor": is_own_anchor, "parentGUID": parent_guid,
@@ -847,7 +947,11 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
         in_data_structure_names = attributes.get('In Data Structure Names', {}).get('name_list', None)
 
         data_class = attributes.get('Data Class', {}).get('value', None)
-        glossary_term = attributes.get('Glossary Term', {}).get('value', None)
+        data_class_guid = attributes.get('Data Class', {}).get('guid', None)
+
+        glossary_term_guid = attributes.get('Glossary Term', {}).get('guid', None)
+        if glossary_term_guid:
+            glossary_term_guid = [glossary_term_guid]
 
         glossary_term_guid = attributes.get('Glossary Term', {}).get('guid', None)
 
@@ -891,7 +995,7 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
 
                 # first update the base data field
                 body = {
-                    "class": "UpdateDataFieldRequestBody", "externalSourceGUID": external_source_guid,
+                    "class": "UpdateElementRequestBody", "externalSourceGUID": external_source_guid,
                     "externalSourceName": external_source_name, "effectiveTime": effective_time,
                     "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing, "properties": {
                         "class": "DataFieldProperties", "qualifiedName": qualified_name, "displayName": display_name,
@@ -924,11 +1028,12 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
 
                 # Sync data field related elements (data structure, parent data fields, terms, data classes
                 sync_data_field_rel_elements(egeria_client, data_structure_guid_list, parent_data_field_guids,
-                                             [glossary_term_guid], data_class, guid, display_name, replace_all_props)
+                                             glossary_term_guid, data_class_guid, guid, display_name, replace_all_props)
                 core_props += f"\n\n## In Data Structure {in_data_structure_names}\n\n"
                 core_props += f"\n\n## Glossary Term \n\n{glossary_term}\n\n"
                 core_props += f"\n\n## Parent Data Field\n\n{parent_data_field_names}\n\n"
-                core_props += "\n___\n\n"
+                core_props += f"\n\n## Data Class\n\n{data_class}\n\n"
+                core_props += "\n_______________________________________________________________________________\n\n"
 
                 # Update data classes
                 logger.success(f"Updated Element `{display_name}`\n\n___")
@@ -943,6 +1048,7 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                 else:
                     # First lets create the data field
                     body = {
+                        "class": "NewElementRequestBody",
                         "properties": {
                             "class": "DataFieldProperties", "qualifiedName": qualified_name,
                             "displayName": display_name, "namespace": namespace, "description": description,
@@ -974,7 +1080,7 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                             for ds_guid in data_structure_guid_list:
                                 # todo This is too naive? - need to better accommodate the relationship
                                 df_body = {
-                                    "class": "MemberDataFieldRequestBody", "properties": {
+                                    "class": "RelationshipRequestBody", "properties": {
                                         "class": "MemberDataFieldProperties", "dataFieldPosition": position,
                                         "minCardinality": min_cardinality, "maxCardinality": max_cardinality,
                                         }
@@ -988,7 +1094,7 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                         if glossary_term:
                             if glossary_term_guid:
                                 glossary_body = {
-                                    "class": "MetadataSourceRequestBody", "externalSourceGUID": external_source_guid,
+                                    "class": "RelationshipRequestBody", "externalSourceGUID": external_source_guid,
                                     "externalSourceName": external_source_name, "effectiveTime": effective_time,
                                     "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing
                                     }
@@ -1008,13 +1114,22 @@ def process_data_field_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                             core_props += f"\n\n## Parent Data Field\n\n{parent_data_field_names}\n\n"
 
                         # Link data class
-                        # if data_class:
-                        #     egeria_client.link_data_class_definition(guid, data_class)
-                        #     msg = f"Adding data class `{data_class}` to data field {display_name}"
-                        #     logger.info(msg)
+                        if data_class:
+                            body = {
+                              "class": "RelationshipRequestBody",
+                              "externalSourceGUID": external_source_guid,
+                              "externalSourceName": external_source_name,
+                              "effectiveTime": effective_time,
+                              "forLineage": for_lineage,
+                              "forDuplicateProcessing": for_duplicate_processing
+                            }
+                            egeria_client.link_data_class_definition(guid, data_class_guid, body)
+                            msg = f"Adding data class `{data_class}` to data field {display_name}"
+                            logger.info(msg)
 
                         logger.success(f"Created Element `{display_name}` with guid `{guid}`")
-                        core_props += "\n====================================================\n\n"
+                        logger.success("=====================================================\n\n")
+                        core_props += "\n___\n\n"
                         return core_props
 
                     else:
@@ -1070,8 +1185,8 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
         external_source_guid = attributes.get('External Source Name', {}).get('guid', None)
         external_source_name = attributes.get('External Source Name', {}).get('value', None)
         effective_time = attributes.get('Effective Time', {}).get('value', None)
-        for_lineage = attributes.get('For Lineage', {}).get('value', None)
-        for_duplicate_processing = attributes.get('For Duplicate Processing', {}).get('value', None)
+        for_lineage = attributes.get('For Lineage', {}).get('value', False)
+        for_duplicate_processing = attributes.get('For Duplicate Processing', {}).get('value', False)
         anchor_guid = attributes.get('Anchor ID', {}).get('guid', None)
         is_own_anchor = attributes.get('Is Own Anchor', {}).get('value', None)
         # parent_id = attributes.get('Parent ID', {}).get('value', None)
@@ -1116,7 +1231,7 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
         glossary_term = attributes.get('Glossary Term', {}).get('value', None)
         glossary_term_guid = attributes.get('Glossary Term', {}).get('guid', None)
 
-        merge_update = attributes.get('Merge Update', {}).get('value', None)
+        merge_update = attributes.get('Merge Update', {}).get('value', True)
 
         position = attributes.get('Position', {}).get('value', None)
         min_cardinality = attributes.get('Minimum Cardinality', {}).get('value', None)
@@ -1131,6 +1246,7 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
 
         glossary_term_guid = attributes.get('Glossary Term', {}).get('guid', None)
 
+
         in_data_dictionary = attributes.get('In Data Dictionary', {}).get('value', None)
         in_data_dictionary_names = attributes.get('In Data Dictionary', {}).get('name_list', None)
         data_dict_guid_list = attributes.get("In Data Dictionary", {}).get("guid_list", None)
@@ -1168,7 +1284,7 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
 
                 # first update the base data class
                 body = {
-                    "class": "UpdateDataClassRequestBody", "externalSourceGUID": external_source_guid,
+                    "class": "UpdateElementRequestBody", "externalSourceGUID": external_source_guid,
                     "externalSourceName": external_source_name, "effectiveTime": effective_time,
                     "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing, "properties": {
                         "class": "DataClassProperties", "qualifiedName": qualified_name, "displayName": display_name,
@@ -1198,8 +1314,8 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                 core_props += f"\n\n## In Data Dictionary\n\n{in_data_dictionary_names}\n\n"
 
                 # Sync data field related elements (data structure, parent data fields, terms, data classes
-                sync_data_class_rel_elements(egeria_client, containing_data_class_guids, [glossary_term_guid],
-                                             specializes_data_class_guid, [glossary_term_guid], guid, display_name,
+                sync_data_class_rel_elements(egeria_client, containing_data_class_guids, glossary_term_guid,
+                                             specializes_data_class_guid, guid, display_name,
                                              replace_all_props)
 
                 core_props += f"\n\n## Glossary Term \n\n{glossary_term}\n\n"
@@ -1219,6 +1335,7 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                 else:
                     # First lets create the data class
                     body = {
+                        "class": "NewElementRequestBody",
                         "properties": {
                             "class": "DataClassProperties", "qualifiedName": qualified_name,
                             "displayName": display_name, "description": description, "namespace": namespace,
@@ -1249,7 +1366,7 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
                         if glossary_term:
                             if glossary_term_guid:
                                 glossary_body = {
-                                    "class": "MetadataSourceRequestBody", "externalSourceGUID": external_source_guid,
+                                    "class": "RelationshipRequestBody", "externalSourceGUID": external_source_guid,
                                     "externalSourceName": external_source_name, "effectiveTime": effective_time,
                                     "forLineage": for_lineage, "forDuplicateProcessing": for_duplicate_processing
                                     }
@@ -1282,7 +1399,7 @@ def process_data_class_upsert_command(egeria_client: EgeriaTech, txt: str, direc
 
 
 @logger.catch
-def process_data_dict_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[str]:
+def process_data_collection_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[str]:
     """
     Processes a Data Dictionary list object_action by extracting key attributes such as
      search string from the given text.
@@ -1293,6 +1410,12 @@ def process_data_dict_list_command(egeria_client: EgeriaTech, txt: str, directiv
     :return: A string summarizing the outcome of the processing.
     """
     command, object_type, object_action = extract_command_plus(txt)
+    if object_type in ["Data Dictionary", "Data Dictionaries", "DataDict", "DataDictionary"]:
+        col_type = "DataDictionary"
+    elif object_type in ["Data Specification", "Data Specifications", "Data Specs"]:
+        col_type = "DataSpec"
+    else:
+        col_type = "Collection"
 
     parsed_output = parse_view_command(egeria_client, object_type, object_action, txt, directive)
 
@@ -1324,13 +1447,197 @@ def process_data_dict_list_command(egeria_client: EgeriaTech, txt: str, directiv
                 logger.error(msg)
                 return None
 
-            list_md = f"\n# Data Dictionaries with filter: `{search_string}`\n\n"
+            list_md = f"\n# `{col_type}` with filter: `{search_string}`\n\n"
+            if search_string == "*":
+                struct = egeria_client.get_classified_collections(col_type, output_format=output_format)
+            else:
+                struct = egeria_client.find_collections(search_string, output_format=output_format)
+
+            if output_format == "DICT":
+                list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
+            else:
+                list_md += struct
+            logger.info(f"Wrote `{col_type}` for search string: `{search_string}`")
+
+            return list_md
+
+        except Exception as e:
+            logger.error(f"Error performing {command}: {e}")
+            console.print_exception(show_locals=True)
+            return None
+    else:
+        return None
+
+def process_data_structure_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[str]:
+    """
+    Processes a Data Dictionary list object_action by extracting key attributes such as
+     search string from the given text.
+
+    :param txt: A string representing the input cell to be processed for
+        extracting term-related attributes.
+    :param directive: an optional string indicating the directive to be used - display, validate or execute
+    :return: A string summarizing the outcome of the processing.
+    """
+    command, object_type, object_action = extract_command_plus(txt)
+
+    parsed_output = parse_view_command(egeria_client, object_type, object_action, txt, directive)
+
+    attributes = parsed_output['attributes']
+
+    valid = parsed_output['valid']
+
+    print(Markdown(parsed_output['display']))
+
+    if directive == "display":
+        return None
+    elif directive == "validate":
+        if valid:
+            print(Markdown(f"==> Validation of {command} completed successfully!\n"))
+        else:
+            msg = f"Validation failed for object_action `{command}`\n"
+            logger.error(msg)
+        return valid
+
+    elif directive == "process":
+        attributes = parsed_output['attributes']
+        search_string = attributes.get('Search String', {}).get('value', '*')
+        output_format = attributes.get('Output Format', {}).get('value', 'LIST')
+        detailed = attributes.get('Detailed', {}).get('value', False)
+
+        try:
+            if not valid:  # First validate the command before we process it
+                msg = f"Validation failed for {object_action} `{object_type}`\n"
+                logger.error(msg)
+                return None
+
+            list_md = f"\n# `{object_type}` with filter: `{search_string}`\n\n"
+            struct = egeria_client.find_data_structures(search_string, output_format=output_format)
+
+            if output_format == "DICT":
+                list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
+            else:
+                list_md += struct
+            logger.info(f"Wrote `{object_type}` for search string: `{search_string}`")
+
+            return list_md
+
+        except Exception as e:
+            logger.error(f"Error performing {command}: {e}")
+            console.print_exception(show_locals=True)
+            return None
+    else:
+        return None
+
+def process_data_field_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[str]:
+    """
+    Processes a Data Dictionary list object_action by extracting key attributes such as
+     search string from the given text.
+
+    :param txt: A string representing the input cell to be processed for
+        extracting term-related attributes.
+    :param directive: an optional string indicating the directive to be used - display, validate or execute
+    :return: A string summarizing the outcome of the processing.
+    """
+    command, object_type, object_action = extract_command_plus(txt)
+
+    parsed_output = parse_view_command(egeria_client, object_type, object_action, txt, directive)
+
+    attributes = parsed_output['attributes']
+
+    valid = parsed_output['valid']
+
+    print(Markdown(parsed_output['display']))
+
+    if directive == "display":
+        return None
+    elif directive == "validate":
+        if valid:
+            print(Markdown(f"==> Validation of {command} completed successfully!\n"))
+        else:
+            msg = f"Validation failed for object_action `{command}`\n"
+            logger.error(msg)
+        return valid
+
+    elif directive == "process":
+        attributes = parsed_output['attributes']
+        search_string = attributes.get('Search String', {}).get('value', '*')
+        output_format = attributes.get('Output Format', {}).get('value', 'LIST')
+        detailed = attributes.get('Detailed', {}).get('value', False)
+
+        try:
+            if not valid:  # First validate the command before we process it
+                msg = f"Validation failed for {object_action} `{object_type}`\n"
+                logger.error(msg)
+                return None
+
+            list_md = f"\n# `{object_type}` with filter: `{search_string}`\n\n"
+            struct = egeria_client.find_data_fields(search_string, output_format=output_format)
+
+            if output_format == "DICT":
+                list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
+            else:
+                list_md += struct
+            logger.info(f"Wrote `{object_type}` for search string: `{search_string}`")
+
+            return list_md
+
+        except Exception as e:
+            logger.error(f"Error performing {command}: {e}")
+            console.print_exception(show_locals=True)
+            return None
+    else:
+        return None
+
+def process_data_class_list_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[str]:
+    """
+    Processes a Data Dictionary list object_action by extracting key attributes such as
+     search string from the given text.
+
+    :param txt: A string representing the input cell to be processed for
+        extracting term-related attributes.
+    :param directive: an optional string indicating the directive to be used - display, validate or execute
+    :return: A string summarizing the outcome of the processing.
+    """
+    command, object_type, object_action = extract_command_plus(txt)
+
+    parsed_output = parse_view_command(egeria_client, object_type, object_action, txt, directive)
+
+    attributes = parsed_output['attributes']
+
+    valid = parsed_output['valid']
+
+    print(Markdown(parsed_output['display']))
+
+    if directive == "display":
+        return None
+    elif directive == "validate":
+        if valid:
+            print(Markdown(f"==> Validation of {command} completed successfully!\n"))
+        else:
+            msg = f"Validation failed for object_action `{command}`\n"
+            logger.error(msg)
+        return valid
+
+    elif directive == "process":
+        attributes = parsed_output['attributes']
+        search_string = attributes.get('Search String', {}).get('value', '*')
+        output_format = attributes.get('Output Format', {}).get('value', 'LIST')
+        detailed = attributes.get('Detailed', {}).get('value', False)
+
+        try:
+            if not valid:  # First validate the command before we process it
+                msg = f"Validation failed for {object_action} `{object_type}`\n"
+                logger.error(msg)
+                return None
+
+            list_md = f"\n# `{object_type}` with filter: `{search_string}`\n\n"
+            struct = egeria_client.find_data_classes(search_string, output_format=output_format)
+
             if output_format == "DICT":
-                struct = egeria_client.get_classified_collections('DataDictionary', output_format=output_format)
-                list_md += f"```{json.dumps(struct, indent=4)}```\n"
+                list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
             else:
-                list_md += egeria_client.find_collections(search_string, output_format=output_format)
-            logger.info(f"Wrote Dictionaries for search string: `{search_string}`")
+                list_md += struct
+            logger.info(f"Wrote `{object_type}` for search string: `{search_string}`")
 
             return list_md