pyegeria 5.4.0.28__py3-none-any.whl → 5.5.3.3__py3-none-any.whl

pyegeria/view/output_formatter.py

@@ -0,0 +1,1297 @@
+"""
+SPDX-License-Identifier: Apache-2.0
+Copyright Contributors to the ODPi Egeria project.
+
+Utilities for rendering Egeria elements into various output formats
+such as DICT, LIST, Markdown (including Mermaid), and basic HTML.
+
+Exceptions
+----------
+Functions in this module may raise the following exceptions:
+- ValueError: Unsupported or inconsistent output format requests (e.g., LIST/TABLE
+  without the required columns); invalid option combinations.
+- KeyError: Missing expected keys from provided elements or report specifications.
+- TypeError: Incorrect input types (e.g., non-list where a list of dicts is required).
+- RuntimeError: Failures in Markdown/HTML conversion or Mermaid rendering helpers.
+"""
+
+import copy
+from datetime import datetime
+import re
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from pyegeria.core._globals import MERMAID_GRAPH_TITLES, MERMAID_GRAPHS
+from pyegeria.core.utils import (camel_to_title_case)
+from markdown_it import MarkdownIt
+from rich.console import Console
+from loguru import logger
+
+from pyegeria.view.mermaid_utilities import construct_mermaid_web
+from pyegeria.view.base_report_formats import select_report_format, MD_SEPARATOR, get_report_spec_match
+from pyegeria.models import to_camel_case
+
+"""
+Note on select_report_spec function:
+
+This function and related data structures have been moved back to _output_formats.py.
+Please import select_report_spec from pyegeria._output_formats instead of from this module.
+"""
+# Todo - put this back after testing
+# console = Console(width=settings.Environment.console_width)
+console = Console(width=300)
+
+
+def _extract_referenceable_properties(element: dict[str, Any]) -> dict[str, Any]:
+    # Get general header attributes
+    guid = element.get('elementHeader', {}).get("guid", None)
+    if guid is None:
+        return {}
+    metadata_collection_id = element['elementHeader']['origin'].get("homeMetadataCollectionId", None)
+    metadata_collection_name = element['elementHeader']['origin'].get("homeMetadataCollectionName", None)
+    origin_category = element['elementHeader'].get("origin_category", None)
+    created_by = element['elementHeader']["versions"].get("createdBy", None)
+    create_time = element['elementHeader']["versions"].get("createTime", None)
+    updated_by = element['elementHeader']["versions"].get("updatedBy", None)
+    version = element['elementHeader']["versions"].get("version", None)
+    type_name = element['elementHeader']["type"].get("typeName", None)
+    classifications = element['elementHeader'].get("classifications", [])
+    effective_from = element['elementHeader'].get("effectiveFrom", None)
+    effective_to = element['elementHeader'].get("effectiveTo", None)
+    status = element['elementHeader'].get("status", None)
+    # Get attributes from properties
+    properties = element['properties']
+    url = properties.get('url', None)
+    display_name = properties.get("displayName","")
+    description = properties.get("description", "") or ""
+    qualified_name = properties.get("qualifiedName", "") or ""
+    category = properties.get("category", "") or ""
+    version_identifier = properties.get("versionIdentifier", "") or ""
+    additional_properties = properties.get("additionalProperties", {}) or {}
+    extended_properties = properties.get("extendedProperties", {}) or {}
+    contentStatus = properties.get("contentStatus", "")
+    activityStatus = properties.get("activityStatus", "")
+    deploymentStatus = properties.get("deploymentStatus", "")
+
+    return {
+        "GUID": guid,
+        "metadata_collection_id": metadata_collection_id,
+        "metadata_collection_name": metadata_collection_name,
+        "origin_category": origin_category,
+        "created_by": created_by,
+        "create_time": create_time,
+        "updated_by": updated_by,
+        "version": version,
+        "type_name": type_name,
+        "classifications": classifications,
+        "element_status": status,
+        "url": url,
+        "display_name": display_name,
+        "description": description,
+        "qualified_name": qualified_name,
+        "category": category,
+        "version_identifier": version_identifier,
+        "additional_properties": additional_properties,
+        "extended_properties": extended_properties,
+        "effective_from": effective_from,
+        "effective_to": effective_to,
+        "content_status": contentStatus,
+        "activity_status": activityStatus,
+        "deployment_status": deploymentStatus,
+        }
+
+
+
+
+
+
+def markdown_to_html(markdown_text: str) -> str:
+    """
+    Convert markdown text to HTML, with special handling for mermaid code blocks.
+
+    Args:
+        markdown_text: The markdown text to convert
+
+    Returns:
+        HTML string
+    """
+    # Initialize markdown-it
+    md = MarkdownIt()
+
+    # Find all mermaid code blocks
+    mermaid_blocks = re.findall(r'```mermaid\n(.*?)\n```', markdown_text, re.DOTALL)
+
+    # Replace each mermaid block with a placeholder
+    placeholders = []
+    for i, block in enumerate(mermaid_blocks):
+        placeholder = f"MERMAID_PLACEHOLDER_{i}"
+        markdown_text = markdown_text.replace(f"```mermaid\n{block}\n```", placeholder)
+        placeholders.append((placeholder, block))
+
+    # Convert markdown to HTML
+    html_text = md.render(markdown_text)
+
+    # Replace placeholders with rendered mermaid HTML
+    for placeholder, mermaid_block in placeholders:
+        mermaid_html = construct_mermaid_web(mermaid_block)
+        html_text = html_text.replace(placeholder, mermaid_html)
+
+    # Add basic HTML structure
+    html_text = f"""
+    <!DOCTYPE html>
+    <html>
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Egeria Report</title>
+        <script src="https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.min.js"></script>
+        <script>
+            mermaid.initialize({{ startOnLoad: true }});
+        </script>
+        <style>
+            body {{ font-family: Arial, sans-serif; line-height: 1.6; padding: 20px; }}
+            h1 {{ color: #2c3e50; }}
+            h2 {{ color: #3498db; }}
+            pre {{ background-color: #f8f8f8; padding: 10px; border-radius: 5px; overflow-x: auto; }}
+            table {{ border-collapse: collapse; width: 100%; margin-bottom: 20px; }}
+            th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
+            th {{ background-color: #f2f2f2; }}
+            tr:nth-child(even) {{ background-color: #f9f9f9; }}
+        </style>
+    </head>
+    <body>
+        {html_text}
+    </body>
+    </html>
+    """
+
+    return html_text
+
+def make_preamble(obj_type: str, search_string: str, output_format: str = 'MD') -> Tuple[str, Optional[str]]:
+    """
+    Creates a preamble string and an elements action based on the given object type, search string,
+    and output format.
+
+    Args:
+        obj_type: The type of object being updated or reported on (e.g., "Product", "Category").
+        search_string: The search string used to filter objects. Defaults to "All Elements" if None.
+        output_format: A format identifier determining the output structure.
+            JSON - output standard json
+            MD - output standard markdown with no preamble
+            FORM - output markdown with a preamble for a form
+            REPORT - output markdown with a preamble for a report
+
+    Returns:
+        tuple: A tuple containing:
+            - A string representing the formatted update or report preamble.
+            - A string or None indicating the action description for the elements,
+              depending on the output format.
+    """
+    # search_string = search_string if search_string else "All Elements"
+    elements_md = ""
+    elements_action = "Update " + obj_type
+    if output_format == "FORM":
+        preamble = f"\n# Update {obj_type} Form - created at {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
+        if search_string:
+            preamble +=  f"\t {obj_type} found from the search string:  `{search_string}`\n\n"
+        return preamble, elements_action
+    elif output_format == "REPORT":
+        elements_md += (f"# {obj_type} Report - created at {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
+                       f"\t{obj_type}  found from the search string:  `{search_string}`\n\n")
+        elements_action = None
+        return elements_md, elements_action
+    else:
+        return "\n", elements_action
+
+def make_md_attribute(attribute_name: str, attribute_value: str, output_type: str) -> Optional[str]:
+    """
+    Create a markdown attribute line for a given attribute name and value.
+
+    Args:
+        attribute_name: The name of the attribute
+        attribute_value: The value of the attribute
+        output_type: The output format (FORM, MD, REPORT)
+
+    Returns:
+        str: Formatted markdown for the attribute
+    """
+    output = ""
+    if isinstance(attribute_value,str):
+        attribute_value = attribute_value.strip() if attribute_value else ""
+    elif isinstance(attribute_value,list) and len(attribute_value) > 0:
+        attribute_value = ",\n".join(attribute_value)
+    if attribute_name:
+        if attribute_name.upper() == "GUID":
+            attribute_title = attribute_name.upper()
+        else:
+            # attribute_title = attribute_name.title()
+            attribute_title = camel_to_title_case(attribute_name)
+    else:
+        attribute_title = ""
+
+    if output_type in ["FORM", "MD"]:
+        if attribute_name.lower() in [ "mermaid", "solutionBlueprintMermaidGraph", "links", "implemented by", "sub_components"]:
+            return '\n'
+
+        output = f"## {attribute_title}\n{attribute_value}\n\n"
+    elif output_type in ["REPORT", "MERMAID"]:
+        if attribute_title in MERMAID_GRAPH_TITLES + ['Mermaid Graph', 'Mermaid']:
+            output = f"## {attribute_title}\n\n```mermaid\n{attribute_value}\n```\n"
+        elif attribute_value:
+            output = f"## {attribute_title}\n{attribute_value}\n\n"
+    return output
+
+def format_for_markdown_table(text: str, guid: str = None) -> str:
+    """
+    Format text for markdown tables by replacing newlines with spaces and escaping pipe characters.
+    No truncation is applied to allow full-length text display regardless of console width.
+
+    Args:
+        text (str): The text to format
+
+    Returns:
+        str: Formatted text safe for markdown tables
+    """
+    if not text:
+        return ""
+    # Replace newlines with spaces and escape pipe characters
+    if isinstance(text, list):
+        text = "\n".join(text)
+    t = text.replace("\n", " ").replace("|", "\\|")
+    if '::' in t and guid:
+        t = f" [{t}](#{guid}) "
+    return t
+
+
+def populate_columns_from_properties(element: dict, columns_struct: dict) -> dict:
+    """
+    Populate a columns_struct with values from the element's properties.
+
+    The element can be either:
+      - a full element dict that contains a nested 'properties' dict (preferred), or
+      - a dict that itself represents the set of properties.
+
+    In both cases, the effective properties are expected to use camelCase keys.
+    The columns_struct is expected to follow the format returned by select_report_spec, where
+    columns are located at columns_struct['formats']['columns'] and each column is a dict containing
+    at least a 'key' field expressed in snake_case. For each column whose snake_case key corresponds
+    to a key in the element properties (after converting to camelCase), this function adds a 'value'
+    entry to the column with the matching property's value.
+
+    Args:
+        element: Either a dict with a nested 'properties' dict using camelCase keys, or
+                 a dict that itself is the set of properties.
+        columns_struct: The columns structure whose columns have snake_case 'key' fields.
+
+    Returns:
+        The updated columns_struct (the input structure is modified in place and also returned).
+    """
+    if not isinstance(columns_struct, dict):
+        return columns_struct
+
+    # Determine the effective properties:
+    # 1) Prefer element['properties'] if present and a dict
+    # 2) Otherwise, if element itself is a dict, treat it as the properties dict
+    # 3) Otherwise, nothing to do
+    props: dict | None = None
+    if isinstance(element, dict):
+        maybe_props = element.get('properties')
+        props = maybe_props if isinstance(maybe_props, dict) else element
+    if not isinstance(props, dict):
+        return columns_struct
+
+    # Get the attributes list if present
+    formats = columns_struct.get('formats') or {}
+    columns = formats.get('attributes') if isinstance(formats, dict) else None
+    if not isinstance(columns, list):
+        return columns_struct
+
+    # Pre-compute snake_case to camelCase mapping for all columns (cache for performance)
+    key_mapping = {}
+    for col in columns:
+        if isinstance(col, dict):
+            key_snake = col.get('key')
+            if key_snake:
+                key_mapping[key_snake] = to_camel_case(key_snake)
+
+    # Single pass to populate values
+    for col in columns:
+        try:
+            key_snake = col.get('key') if isinstance(col, dict) else None
+            if not key_snake:
+                continue
+            # Use pre-computed camelCase key
+            key_camel = key_mapping.get(key_snake)
+            if key_camel and key_camel in props:
+                col['value'] = props.get(key_camel)
+        except Exception as e:
+            # Be resilient; log and continue
+            logger.debug(f"populate_columns_from_properties: skipping column due to error: {e}")
+            continue
+
+    return columns_struct
+
+def populate_feedback(element: dict, columns_struct: dict) -> dict:
+    """
+    Populate a columns_struct with values from the element's feedback.
+
+    The element dict may have feedback relationships. If these are present in the column request, extract them
+    and add the values to the columns_struct.
+    The columns_struct is expected to follow the format returned by select_report_spec, where
+    columns are located at columns_struct['formats']['columns'] and each column is a dict containing
+    at least a 'key' field expressed in snake_case. For each column whose snake_case key corresponds
+    to a key in the element properties (after converting to camelCase), this function adds a 'value'
+    entry to the column with the matching property's value.
+
+    Args:
+        element: The element containing a 'properties' dict with camelCase keys.
+        columns_struct: The columns structure whose columns have snake_case 'key' fields.
+
+    Returns:
+        The updated columns_struct (the input structure is modified in place and also returned).
+    """
+    if not isinstance(columns_struct, dict):
+        return columns_struct
+
+    # Need to think more about what I want to produce
+    return columns_struct
+    props = {}
+
+    # Get the attributes list if present
+    formats = columns_struct.get('formats') or {}
+    columns = formats.get('attributes') if isinstance(formats, dict) else None
+    if not isinstance(columns, list):
+        return columns_struct
+
+    for col in columns:
+        try:
+            key_snake = col.get('key') if isinstance(col, dict) else None
+            if not key_snake:
+                continue
+            # Convert the snake_case key to camelCase to look up in properties
+            key_camel = to_camel_case(key_snake)
+            if key_camel in props:
+                col['value'] = props.get(key_camel)
+        except Exception as e:
+            # Be resilient; log and continue
+            logger.debug(f"populate_columns_from_properties: skipping column due to error: {e}")
+            continue
+
+    return columns_struct
+
+def get_required_relationships(element: dict, columns_struct: dict) -> dict:
+    """
+    Populate relationship-derived column values in columns_struct based on top-level keys in the element.
+    
+    NOTE: This function is now a lightweight wrapper around the optimized logic in populate_common_columns.
+    For best performance, use populate_common_columns directly with include_relationships=True.
+
+    This function inspects the requested columns in columns_struct, converts each column key from
+    snake_case to camelCase, and if a matching top-level key exists in the element, parses that value
+    (typically lists of relationship beans) into a human-readable value (e.g., a comma-separated list
+    of qualified names) and stores it under the column's 'value'. Columns not specified in the
+    columns_struct are ignored. Existing non-empty 'value's are left as-is.
+
+    Example: if a column with key 'member_of_collections' is present, this function will look for the
+    top-level key 'memberOfCollections' in the element and derive a value if found.
+
+    Args:
+        element: The element dictionary containing top-level relationship lists (e.g., associatedGlossaries,
+                 memberOfCollections, collectionMembers).
+        columns_struct: The columns structure to augment with derived 'value's.
+
+    Returns:
+        The updated columns_struct (modified in place and returned).
+    """
+    if not isinstance(columns_struct, dict):
+        return columns_struct
+
+    formats = columns_struct.get('formats') or {}
+    columns = formats.get('attributes') if isinstance(formats, dict) else None
+    if not isinstance(columns, list):
+        return columns_struct
+
+    # Pre-compute relationship values for efficiency
+    relationship_values = {}
+    for col in columns:
+        if not isinstance(col, dict):
+            continue
+        key_snake = col.get('key')
+        if not key_snake:
+            continue
+        # If already has a non-empty value, don't overwrite
+        if col.get('value') not in (None, ""):
+            continue
+            
+        key_camel = to_camel_case(key_snake)
+        if key_camel not in element:
+            continue
+
+        top_val = element.get(key_camel)
+        derived_value = ""
+        if isinstance(top_val, list):
+            names = []
+            for item in top_val:
+                nm = _extract_name_from_relationship_item(item)
+                if nm:
+                    names.append(nm)
+            derived_value = ", ".join(names)
+        elif isinstance(top_val, dict):
+            nm = _extract_name_from_relationship_item(top_val)
+            derived_value = nm or ""
+        else:
+            derived_value = str(top_val) if top_val is not None else ""
+        
+        if derived_value:
+            relationship_values[key_snake] = derived_value
+
+    # Apply the values
+    for col in columns:
+        if isinstance(col, dict):
+            key = col.get('key')
+            if key and key in relationship_values and col.get('value') in (None, ""):
+                col['value'] = relationship_values[key]
+
+    return columns_struct
+
+
+
+
+def generate_entity_md(elements: List[Dict], 
+                      elements_action: str, 
+                      output_format: str, 
+                      entity_type: str,
+                      extract_properties_func: Callable, 
+                      get_additional_props_func: Optional[Callable] = None,
+                       columns_struct: [dict] = None) -> str:
+    """
+    Generic method to generate markdown for entities.
+
+    Args:
+        elements (list): List of entity elements
+        elements_action (str): Action description for elements
+        output_format (str): Output format
+        entity_type (str): Type of entity (Glossary, Term, Category, etc.)
+        extract_properties_func: Function to extract properties from an element
+        get_additional_props_func: Optional function to get additional properties
+        columns (list): List of column name structures
+
+    Returns:
+        str: Markdown representation
+    """
+    heading = columns_struct.get("heading")
+    if heading == "Default Base Attributes":
+        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of report_spec and output_format?\n\n"
+    else:
+        elements_md = ""
+    base_columns = columns_struct['formats'].get('attributes') if columns_struct else None
+
+    for element in elements:
+        if element is None:
+                continue
+        guid = element.get('elementHeader', {}).get('guid')
+
+        # Prefer new behavior: extractor returns an updated columns_struct with values
+        returned_struct = None
+        if columns_struct is not None:
+            try:
+                returned_struct = extract_properties_func(element, columns_struct)
+            except TypeError:
+                # Fallback for legacy extractors without columns_struct parameter
+                returned_struct = None
+        
+        # Legacy fallback: get props dict if no columns_struct provided/returned
+        props = {}
+        if returned_struct is None:
+            props = extract_properties_func(element) if callable(extract_properties_func) else {}
+
+        # Get additional properties if function is provided
+        additional_props = {}
+        if get_additional_props_func:
+            # Use guid if available, else try to get from props
+            guid_for_fmt = guid or props.get('GUID')
+            additional_props = get_additional_props_func(element, guid_for_fmt, output_format)
+
+        # Determine display name
+        display_name = None
+        if returned_struct is not None:
+            cols = returned_struct.get('formats', {}).get('attributes', [])
+            # Find value from 'display_name' or 'title'
+            for col in cols:
+                if col.get('key') in ('display_name', 'title', 'keyword'):
+                    display_name = col.get('value')
+                    if display_name:
+                        break
+        else:
+            display_name = props.get('display_name') or props.get('title') or props.get('keyword')
+
+        if display_name is None and (keyword:= element.get('properties',{}).get('keyword',None)) is not None:
+            display_name = keyword
+        elif display_name is None:
+            display_name = "NO DISPLAY NAME"
+
+        # Format header based on output format
+        if output_format in ['FORM', 'MD']:
+            elements_md += f"# {elements_action}\n\n"
+            elements_md += f"## {entity_type} Name \n\n{display_name}\n\n"
+        elif output_format == 'REPORT':
+            elements_md += f'<a id="{(guid or props.get("GUID") or "No GUID" )}"></a>\n# {entity_type} Name: {display_name}\n\n'
+        else:
+            elements_md += f"## {entity_type} Name \n\n{display_name}\n\n"
+
+        # Add attributes based on column spec if available, otherwise, add all (legacy)
+        if returned_struct is not None:
+            cols = returned_struct.get('formats', {}).get('attributes', [])
+            for column in cols:
+                name = column.get('name')
+                key = column.get('key')
+                value = column.get('value')
+                if value in (None, "") and key in additional_props:
+                    value = additional_props[key]
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid)
+                elements_md += make_md_attribute(name, value, output_format)
+            if wk := returned_struct.get("annotations", {}).get("wikilinks", None):
+                elements_md += ", ".join(wk)
+        elif base_columns:
+            # If we have columns but extractor didn't return struct, use legacy props lookup
+            for column in base_columns:
+                key = column['key']
+                name = column['name']
+                value = ""
+                if key in props:
+                    value = props[key]
+                elif key in additional_props:
+                    value = additional_props[key]
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid or props.get('GUID'))
+                elements_md += make_md_attribute(name, value, output_format)
+            if wk := columns_struct.get("annotations", {}).get("wikilinks", None):
+                elements_md += ", ".join(wk)
+        else:
+            # Legacy path without columns: dump all props
+            for key, value in props.items():
+                if output_format in ['FORM', 'MD', 'DICT'] and key == 'mermaid':
+                    continue
+                if key not in ['properties', 'display_name']:
+                    if key == 'mermaid' and value == '':
+                        continue
+                    elements_md += make_md_attribute(key.replace('_', ' '), value, output_format)
+            for key, value in additional_props.items():
+                elements_md += make_md_attribute(key.replace('_', ' '), value, output_format)
+
+        if element != elements[-1]:
+            elements_md += MD_SEPARATOR
+
+    return elements_md
+
+def generate_entity_md_table(elements: List[Dict],
+                            search_string: str,
+                            entity_type: str,
+                            extract_properties_func: Callable,
+                            columns_struct: dict,
+                            get_additional_props_func: Optional[Callable] = None,
+                            output_format: str = 'LIST') -> str:
+    """
+    Generic method to generate a markdown table for entities.
+
+    Args:
+        elements (list): List of entity elements
+        search_string (str): The search string used
+        entity_type (str): Type of entity (Glossary, Term, Category, etc.)
+        extract_properties_func: Function to extract properties from an element
+        columns: List of column definitions, each containing 'name', 'key', and 'format' (optional)
+        get_additional_props_func: Optional function to get additional properties
+        output_format (str): Output format (FORM, REPORT, LIST, etc.)
+
+    Returns:
+        str: Markdown table
+    """
+    # Handle pluralization - if entity_type ends with 'y', use 'ies' instead of 's'
+    target_type = columns_struct.get('target_type', entity_type)
+    # if target_type.endswith('y'):
+    #     target_type = target_type.replace('y', 'ies')
+    # else:
+    #     target_type = target_type.replace('s', 's')
+
+    entity_type_plural = f"{target_type[:-1]}ies" if target_type.endswith('y') else f"{target_type}s"
+    # entity_type_plural = target_type
+    columns = columns_struct['formats'].get('attributes', [])
+    heading = columns_struct.get("heading")
+    if heading == "Default Base Attributes":
+        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of report_spec and output_format?\n\n"
+    else:
+        elements_md = ""
+
+    if output_format == "LIST":
+        elements_md = f"# {entity_type_plural} Table\n\n"
+        elements_md += f"{entity_type_plural} found from the search string: `{search_string}`\n\n"
+
+    # Add column headers
+    header_row = "| "
+    separator_row = "|"
+    for column in columns:
+        header_row += f"{column['name']} | "
+        separator_row += "-------------|"
+
+    elements_md += header_row + "\n"
+    elements_md += separator_row + "\n"
+
+    for element in elements:
+        guid = element.get('elementHeader', {}).get('guid', None)
+
+        # Extractor returns columns_struct with values when possible
+        # Use shallow copy and reset only column values for performance
+        local_columns_struct = columns_struct.copy()
+        if 'formats' in local_columns_struct and 'attributes' in local_columns_struct['formats']:
+            # Create new list of columns with reset values
+            local_columns_struct['formats'] = local_columns_struct['formats'].copy()
+            local_columns_struct['formats']['attributes'] = [
+                {**col, 'value': None} if isinstance(col, dict) else col
+                for col in local_columns_struct['formats']['attributes']
+            ]
+        try:
+            returned_struct = extract_properties_func(element, local_columns_struct)
+        except TypeError:
+            returned_struct = None
+
+        # For help mode, bypass extraction
+        if output_format == "help":
+            returned_struct = {"formats": {"attributes": columns}}
+
+        # Additional props (if any)
+        additional_props = {}
+        if get_additional_props_func:
+            additional_props = get_additional_props_func(element, guid, output_format)
+
+        # Build row
+        row = "| "
+        if returned_struct is not None:
+            for column in returned_struct.get('formats', {}).get('attributes', []):
+                key = column.get('key')
+                value = column.get('value')
+                if (value in (None, "")) and key in additional_props:
+                    value = additional_props[key]
+
+                if value in (None, ""):
+                    value = " --- "
+
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid)
+                elif isinstance(value, str):
+                    value = value.replace("\n", " ").replace("|", "\\|")
+
+                row += f"{value} | "
+        else:
+            # Legacy fallback: read from props dict
+            props = extract_properties_func(element)
+            for column in columns:
+                key = column['key']
+                value = " "
+                if key in props:
+                    value = props[key]
+                elif key in additional_props:
+                    value = additional_props[key]
+
+                if value in (None, "", " "):
+                    value = " --- "
+
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid or props.get('GUID'))
+                elif isinstance(value, str):
+                    value = value.replace("\n", " ").replace("|", "\\|")
+
+                row += f"{value} | "
+
+        elements_md += row + "\n"
+        # if wk := columns_struct.get("annotations",{}).get("wikilinks", None):
+        #     elements_md += ", ".join(wk)
+    return elements_md
+
+
+def generate_entity_dict(elements: List[Dict], 
+                        extract_properties_func: Callable, 
+                        get_additional_props_func: Optional[Callable] = None,
+                        include_keys: Optional[List[str]] = None, 
+                        exclude_keys: Optional[List[str]] = None,
+                        columns_struct: dict = None,
+                        output_format: str = 'DICT') -> List[Dict]:
+    """
+    Generic method to generate a dictionary representation of entities.
+
+    Args:
+        elements (list): List of entity elements
+        extract_properties_func: Function to extract properties from an element
+        get_additional_props_func: Optional function to get additional properties
+        include_keys: Optional list of keys to include in the result (if None, include all)
+        exclude_keys: Optional list of keys to exclude from the result (if None, exclude none)
+        columns_struct: Optional dict of columns to include (if None, include all)
+        output_format (str): Output format (FORM, REPORT, DICT, etc.)
+
+    Returns:
+        list: List of entity dictionaries
+    """
+    result = []
+
+    #####
+    # Add attributes based on column spec if available, otherwise, add all
+    import copy
+    for element in elements:
+        if not isinstance(element, dict):
+            continue
+
+        guid = element.get('elementHeader', {}).get('guid')
+
+        # Use shallow copy and reset only column values for performance
+        local_columns_struct = None
+        if columns_struct is not None:
+            local_columns_struct = columns_struct.copy()
+            if 'formats' in local_columns_struct and 'attributes' in local_columns_struct['formats']:
+                # Create new list of columns with reset values
+                local_columns_struct['formats'] = local_columns_struct['formats'].copy()
+                local_columns_struct['formats']['attributes'] = [
+                    {**col, 'value': None} if isinstance(col, dict) else col
+                    for col in local_columns_struct['formats']['attributes']
+                ]
+
+        returned_struct = None
+        if local_columns_struct is not None:
+            try:
+                returned_struct = extract_properties_func(element, local_columns_struct)
+            except TypeError as e:
+                logger.info(f"Error - didn't find extractor?: {e}")
+                returned_struct = None
+
+        # Get additional properties if function is provided
+        additional_props = {}
+        if get_additional_props_func:
+            additional_props = get_additional_props_func(element, guid, output_format)
+
+        # Create entity dictionary
+        entity_dict = {}
+
+        cols = local_columns_struct['formats'].get('attributes', None) if local_columns_struct else None
+        if returned_struct is not None:
+            for column in returned_struct.get('formats', {}).get('attributes', []):
+                key = column.get('key')
+                name = column.get('name')
+                value = column.get('value')
+                if (value in (None, "")) and key in additional_props:
+                    value = additional_props[key]
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid)
+                # Avoid overwriting when multiple columns share the same display name in a spec
+                dict_key = name
+                if dict_key in entity_dict:
+                    logger.warning(f"DICT key collision for display name '{dict_key}'. Suffixing duplicate to preserve all values.")
+                    suffix_idx = 1
+                    tmp_key = f"{dict_key}_{suffix_idx}"
+                    while tmp_key in entity_dict:
+                        suffix_idx += 1
+                        tmp_key = f"{dict_key}_{suffix_idx}"
+                    dict_key = tmp_key
+                entity_dict[dict_key] = value
+        elif cols:
+            for column in cols:
+                key = column['key']
+                name = column['name']
+                value = ""
+                props = extract_properties_func(element, columns_struct)
+                if key in props:
+                    value = props[key]
+                elif key in additional_props:
+                    value = additional_props[key]
+                if  column.get('format', None):
+                    value = format_for_markdown_table(value, guid or props.get('GUID'))
+                dict_key = name
+                if dict_key in entity_dict:
+                    logger.warning(f"DICT key collision for display name '{dict_key}'. Suffixing duplicate to preserve all values.")
+                    suffix_idx = 1
+                    tmp_key = f"{dict_key}_{suffix_idx}"
+                    while tmp_key in entity_dict:
+                        suffix_idx += 1
+                        tmp_key = f"{dict_key}_{suffix_idx}"
+                    dict_key = tmp_key
+                entity_dict[dict_key] = value
+        else:
+            props = extract_properties_func(element, local_columns_struct)
+            # Add properties based on include/exclude lists
+            for key, value in props.items():
+                if key not in ['properties', 'mermaid']:  # Skip the raw properties object
+                    if (include_keys is None or key in include_keys) and (
+                            exclude_keys is None or key not in exclude_keys):
+                        entity_dict[key] = value
+
+            # Add additional properties
+            for key, value in additional_props.items():
+                if (include_keys is None or key in include_keys) and (exclude_keys is None or key not in exclude_keys):
+                    entity_dict[key] = value
+
+        result.append(entity_dict)
+    #####
+    # for element in elements:
+    #     if element is None:
+    #         continue
+    #     props = extract_properties_func(element)
+    #
+    #     # Get additional properties if function is provided
+    #     additional_props = {}
+    #     if get_additional_props_func:
+    #         additional_props = get_additional_props_func(element,props['GUID'], output_format)
+    #
+    #     # Create entity dictionary
+    #     entity_dict = {}
+    #
+    #     # Add properties based on include/exclude lists
+    #     for key, value in props.items():
+    #         if key not in [ 'properties', 'mermaid']:  # Skip the raw properties object
+    #             if (include_keys is None or key in include_keys) and (
+    #                     exclude_keys is None or key not in exclude_keys):
+    #                 entity_dict[key] = value
+    #
+    #     # Add additional properties
+    #     for key, value in additional_props.items():
+    #         if (include_keys is None or key in include_keys) and (exclude_keys is None or key not in exclude_keys):
+    #             entity_dict[key] = value
+    #
+    #     result.append(entity_dict)
+
+    return result
+
+def resolve_output_formats(entity_type: str,
+                           output_format: str,
+                           report_spec: Optional[Union[str, dict]] = None,
+                           default_label: Optional[str] = None,
+                           **kwargs) -> Optional[dict]:
+    """
+    Resolve a report format structure given an entity type, the desired output format
+    (e.g., DICT, LIST, MD, REPORT, FORM), and either a label (str) or a dict of format sets.
+
+    Notes:
+    - Preferred naming is report_spec (formerly output_format); compatibility for the
+      old 'output_format_spec' terminology has been removed.
+
+    Selection order:
+    - If report_spec is a str: select by label.
+    - If report_spec is a dict: use get_report_spec_match to pick a matching format.
+    - Else: try selecting by entity_type or default_label.
+    - Fallback: select "Default".
+    """
+    from pyegeria.view.base_report_formats import get_report_spec_match
+
+    if report_spec is None and isinstance(kwargs, dict):
+        if 'report_spec' in kwargs:
+            report_spec = kwargs.get('report_spec')
+        elif 'report_format' in kwargs:
+            # Still accept 'report_format' as a synonym for report_spec
+            report_spec = kwargs.get('report_format')
+
+    if isinstance(report_spec, str):
+        return select_report_format(report_spec, output_format)
+    if isinstance(report_spec, dict):
+        return get_report_spec_match(report_spec, output_format)
+
+    label = default_label or entity_type
+    fmt = select_report_format(label, output_format)
+    if fmt is None:
+        fmt = select_report_format("Default", output_format)
+    return fmt
+
+
+def overlay_additional_values(columns_struct: dict, extra: Optional[dict]) -> dict:
+    """
+    Overlay extra values into columns_struct only where the column's value is empty or missing.
+    Returns the modified columns_struct.
+    """
+    if not isinstance(columns_struct, dict) or not extra:
+        return columns_struct
+    columns = columns_struct.get('formats', {}).get('attributes')
+    if not isinstance(columns, list):
+        return columns_struct
+    for col in columns:
+        if not isinstance(col, dict):
+            continue
+        key = col.get('key')
+        if not key:
+            continue
+        if col.get('value') in (None, "") and key in extra:
+            col['value'] = extra[key]
+    return columns_struct
+
+
+def populate_common_columns(
+    element: dict,
+    columns_struct: dict,
+    *,
+    include_header: bool = True,
+    include_relationships: bool = True,
+    include_subject_area: bool = True,
+    mermaid_source_key: str = 'mermaidGraph',
+    mermaid_dest_key: str = 'mermaid'
+) -> dict:
+    """
+    Populate the common columns in columns_struct based on a standard Egeria element shape.
+
+    Steps:
+    - Populate from element.properties (camelCase mapped from snake_case keys)
+    - Optionally overlay header-derived values (GUID, type_name, times, etc.)
+    - Optionally populate relationship-based columns via get_required_relationships
+    - Optionally populate subject_area from element.elementHeader.subjectArea.classificationProperties.subjectAreaName
+    - If a column with key == mermaid_dest_key is present, set it from mermaid_source_key
+    - Do not overwrite non-empty values already set
+    """
+    # Pre-compute values that will be reused
+    props = element.get('properties', {}) if isinstance(element, dict) else {}
+    header_props = _extract_referenceable_properties(element) if include_header else {}
+    guid = header_props.get('GUID') if include_header else None
+    
+    # Pre-compute subject area value
+    subject_area_val = ""
+    if include_subject_area:
+        try:
+            subject_area = element.get('elementHeader', {}).get('subjectArea') or ""
+            if isinstance(subject_area, dict):
+                subject_area_val = subject_area.get('classificationProperties', {}).get('subjectAreaName', '')
+        except Exception as e:
+            logger.debug(f"populate_common_columns: subject_area handling error: {e}")
+    
+    # Pre-compute mermaid value
+    mermaid_val = element.get(mermaid_source_key, '') or ''
+    
+    # Pre-compute relationship values for efficiency
+    relationship_values = {}
+    if include_relationships:
+        formats = columns_struct.get('formats') or {}
+        columns = formats.get('attributes') if isinstance(formats, dict) else None
+        if isinstance(columns, list):
+            for col in columns:
+                if not isinstance(col, dict):
+                    continue
+                key_snake = col.get('key')
+                if not key_snake:
+                    continue
+                key_camel = to_camel_case(key_snake)
+                if key_camel in element:
+                    top_val = element.get(key_camel)
+                    derived_value = ""
+                    if isinstance(top_val, list):
+                        names = []
+                        for item in top_val:
+                            nm = _extract_name_from_relationship_item(item)
+                            if nm:
+                                names.append(nm)
+                        derived_value = ", ".join(names)
+                    elif isinstance(top_val, dict):
+                        nm = _extract_name_from_relationship_item(top_val)
+                        derived_value = nm or ""
+                    else:
+                        derived_value = str(top_val) if top_val is not None else ""
+                    if derived_value:
+                        relationship_values[key_snake] = derived_value
+    
+    # Single pass through columns - consolidate all operations
+    col_data = columns_struct
+    columns_list = col_data.get('formats', {}).get('attributes', [])
+    
+    for column in columns_list:
+        if not isinstance(column, dict):
+            continue
+        
+        key = column.get('key')
+        if not key:
+            continue
+        
+        # Skip if already has a value
+        if column.get('value') not in (None, ""):
+            continue
+        
+        # 1) Try properties (camelCase conversion)
+        key_camel = to_camel_case(key)
+        if key_camel in props:
+            column['value'] = props.get(key_camel)
+            continue
+        
+        # 2) Try header properties
+        if include_header:
+            if key in header_props:
+                column['value'] = header_props.get(key)
+                continue
+            elif isinstance(key, str) and key.lower() == 'guid':
+                column['value'] = guid
+                continue
+        
+        # 3) Try relationship values
+        if include_relationships and key in relationship_values:
+            column['value'] = relationship_values[key]
+            continue
+        
+        # 4) Try subject area
+        if include_subject_area and key == 'subject_area':
+            column['value'] = subject_area_val
+            continue
+        
+        # 5) Try mermaid
+        if key == mermaid_dest_key:
+            column['value'] = mermaid_val
+            continue
+        if key in MERMAID_GRAPHS and mermaid_dest_key not in MERMAID_GRAPHS:
+            column['value'] = element.get(key, '')
+            continue
+    
+    # Get any Feedback Values that have been requested (kept separate as it may have side effects)
+    col_data = populate_feedback(element, col_data)
+    
+    return col_data
+
+
+def _extract_name_from_relationship_item(item: Any) -> Optional[str]:
+    """Best-effort extraction of a display/qualified name from a relationship item."""
+    try:
+        if isinstance(item, dict):
+            # Common pattern: item['relatedElement']['properties']['qualifiedName']
+            related = item.get('relatedElement') or item.get('related_element')
+            if isinstance(related, dict):
+                props = related.get('properties') or {}
+                name = (
+                    props.get('qualifiedName')
+                    or props.get('displayName')
+                    or props.get('name')
+                )
+                if name:
+                    return name
+            # Sometimes the properties may be at the top level of the item
+            name = (
+                item.get('qualifiedName')
+                or item.get('displayName')
+                or item.get('name')
+            )
+            if name:
+                return name
+        elif isinstance(item, str):
+            return item
+    except Exception as e:
+        logger.debug(f"_extract_name_from_relationship_item: error extracting name: {e}")
+    return None
+
+
+def extract_mermaid_only(elements: Union[Dict, List[Dict]]) -> Union[str, List[str]]:
+    """
+    Extract mermaid graph data from elements.
+
+    Args:
+        elements: Dictionary or list of dictionaries containing element data
+
+    Returns:
+        String or list of strings containing mermaid graph data
+    """
+    if isinstance(elements, dict):
+        mer = elements.get('mermaidGraph', None)
+        if mer:
+            return f"\n```mermaid\n{mer}\n```"
+        else:
+            return "---"
+
+
+    result = []
+    for element in elements:
+        mer = element.get('mermaidGraph', "---")
+        mer_out = f"\n\n```mermaid\n{mer}\n```" if mer else "---"
+        result.append(mer_out)
+    return result
+
+def extract_basic_dict(elements: Union[Dict, List[Dict]]) -> Union[Dict, List[Dict]]:
+    """
+    Extract basic dictionary data from elements.
+
+    Args:
+        elements: Dictionary or list of dictionaries containing element data
+
+    Returns:
+        Dictionary or list of dictionaries with extracted data
+    """
+    if isinstance(elements, dict):
+        body = {'guid': elements['elementHeader']['guid']}
+        for key in elements['properties']:
+            body[key] = elements['properties'][key]
+
+        # Add classifications if present
+        classifications = elements['elementHeader'].get('classifications', [])
+        if classifications:
+            classification_names = "["
+            for classification in classifications:
+                if len(classification_names) > 1:
+                    classification_names += ", "
+                classification_names += f"{classification['classificationName']}"
+            body['classification_names'] = classification_names + ']'
+
+        return body
+
+    result = []
+    for element in elements:
+        if element is None:
+            continue
+        body = {'guid': element['elementHeader']['guid']}
+        for key in element['properties']:
+            body[key] = element['properties'][key]
+
+        # Add classifications if present
+        classifications = element['elementHeader'].get('classifications', [])
+        if classifications:
+            classification_names = "["
+            for classification in classifications:
+                if len(classification_names) > 1:
+                    classification_names += ", "
+                classification_names += f"{classification['classificationName']}"
+            body['classifications'] = classification_names + ']'
+
+        result.append(body)
+    return result
+
+def _extract_default_properties(self, element: dict, columns_struct: dict) -> dict:
+    props = element.get('properties', {}) or {}
+    normalized = {
+        'properties': props,
+        'elementHeader': element.get('elementHeader', {}),
+    }
+    # Common population pipeline
+    col_data = populate_common_columns(element, columns_struct)
+    columns_list = col_data.get('formats', {}).get('attributes', [])
+
+    return col_data
+
+
+def _generate_default_output(self, elements: dict | list[dict], search_string: str,
+                             element_type_name: str | None,
+                             output_format: str = 'DICT',
+                             report_format: dict | str | None = None,
+                             **kwargs) -> str | list[dict]:
+    entity_type = 'Referenceable' if element_type_name is None else element_type_name
+    # Backward compatibility: accept legacy kwarg
+    if report_format is None and isinstance(kwargs, dict) and 'report_spec' in kwargs:
+        report_format = kwargs.get('report_spec')
+    if report_format:
+        if isinstance(report_format, str):
+            output_formats = select_report_format(report_format, output_format)
+        elif isinstance(report_format, dict):
+            output_formats = get_report_spec_match(report_format, output_format)
+        else:
+            output_formats = None
+    else:
+        output_formats = select_report_format(entity_type, output_format)
+    if output_formats is None:
+        output_formats = select_report_format('Default', output_format)
+    return generate_output(
+        elements=elements,
+        search_string=search_string,
+        entity_type=entity_type,
+        output_format=output_format,
+        extract_properties_func=_extract_default_properties,
+        get_additional_props_func=None,
+        columns_struct=output_formats,
+    )
+
+
+def generate_output(elements: Union[Dict, List[Dict]],
+               search_string: str,
+               entity_type: str,
+               output_format: str,
+               extract_properties_func: Callable,
+               get_additional_props_func: Optional[Callable] = None,
+               columns_struct: dict = None) -> Union[str, list[dict]]:
+    """
+    Generate output in the specified format for the given elements.
+
+    Args:
+        elements: Dictionary or list of dictionaries containing element data
+        search_string: The search string used to find the elements
+        entity_type: The type of entity (e.g., "Glossary", "Term", "Category")
+        output_format: The desired output format (MD, FORM, REPORT, LIST, DICT, MERMAID, HTML)
+        extract_properties_func: Function to extract properties from an element
+        get_additional_props_func: Optional function to get additional properties
+        columns: Optional list of column definitions for table output
+
+    Returns:
+        Formatted output as string or list of dictionaries
+    """
+    columns = columns_struct['formats'].get('attributes',None) if columns_struct else None
+    if not columns:
+        columns_struct = select_report_format("Default",output_format)
+        if columns_struct:
+            columns = columns_struct.get('formats', {}).get('attributes', None)
+
+    target_type = columns_struct.get('target_type', entity_type) if columns_struct else entity_type
+    if target_type is None:
+        target_type = entity_type
+
+    # Ensure elements is a list
+    if isinstance(elements, dict):
+        elements = [elements]
+
+    # Handle empty search string
+    if search_string is None or search_string == '':
+        search_string = "All"
+
+    # Set the output format to DICT to return values to table display
+    # if output_format == "TABLE":
+    #     output_format = "DICT"
+
+    # Generate output based on format
+    if output_format == 'MERMAID':
+        return extract_mermaid_only(elements)
+
+    elif output_format == 'HTML':
+        # First generate the REPORT format output
+        report_output = generate_output(
+            elements=elements,
+            search_string=search_string,
+            entity_type=entity_type,
+            output_format="REPORT",
+            extract_properties_func=extract_properties_func,
+            get_additional_props_func=get_additional_props_func,
+            columns_struct=columns_struct
+        )
+
+        # Convert the markdown to HTML
+        return markdown_to_html(report_output)
+
+    elif output_format in ['DICT','TABLE']:
+        return generate_entity_dict(
+            elements=elements,
+            extract_properties_func=extract_properties_func,
+            get_additional_props_func=get_additional_props_func,
+            exclude_keys=['properties'],
+            columns_struct=columns_struct,
+            output_format=output_format
+        )
+
+    elif output_format == 'LIST':
+        if columns is None:
+            raise ValueError("Columns must be provided for LIST output format")
+
+        return generate_entity_md_table(
+            elements=elements,
+            search_string=search_string,
+            entity_type=entity_type,
+            extract_properties_func=extract_properties_func,
+            columns_struct=columns_struct,
+            get_additional_props_func=get_additional_props_func,
+            output_format=output_format
+        )
+
+    else:  #  MD, FORM, REPORT
+        elements_md, elements_action = make_preamble(
+            obj_type=target_type,
+            search_string=search_string,
+            output_format=output_format
+        )
+
+        elements_md += generate_entity_md(
+            elements=elements,
+            elements_action=elements_action,
+            output_format=output_format,
+            entity_type=target_type,
+            extract_properties_func=extract_properties_func,
+            get_additional_props_func=get_additional_props_func,
+            columns_struct = columns_struct
+        )
+
+        return elements_md