pyegeria 5.4.8__py3-none-any.whl → 5.4.8.3__py3-none-any.whl

pyegeria/output_formatter_with_machine_keys.py

@@ -0,0 +1,1127 @@
+from datetime import datetime
+import re
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from pyegeria.utils import (camel_to_title_case)
+from markdown_it import MarkdownIt
+from rich.console import Console
+from loguru import logger
+from pyegeria.config import settings
+
+from pyegeria.mermaid_utilities import construct_mermaid_web
+from pyegeria.base_report_formats import select_report_format, MD_SEPARATOR, get_report_spec_match
+from pyegeria.models import to_camel_case
+
+"""
+Note on select_report_spec function:
+
+This function and related data structures have been moved back to _output_formats.py.
+Please import select_report_spec from pyegeria._output_formats instead of from this module.
+"""
+
+console = Console(width=settings.Environment.console_width)
+
+
+def _extract_referenceable_properties(element: dict[str, Any]) -> dict[str, Any]:
+    # Get general header attributes
+    guid = element.get('elementHeader', {}).get("guid", None)
+    if guid is None:
+        return {}
+    metadata_collection_id = element['elementHeader']['origin'].get("homeMetadataCollectionId", None)
+    metadata_collection_name = element['elementHeader']['origin'].get("homeMetadataCollectionName", None)
+    origin_category = element['elementHeader'].get("origin_category", None)
+    created_by = element['elementHeader']["versions"].get("createdBy", None)
+    create_time = element['elementHeader']["versions"].get("createTime", None)
+    updated_by = element['elementHeader']["versions"].get("updatedBy", None)
+    version = element['elementHeader']["versions"].get("version", None)
+    type_name = element['elementHeader']["type"].get("typeName", None)
+    classifications = element['elementHeader'].get("classifications", [])
+    effective_from = element['elementHeader'].get("effectiveFrom", None)
+    effective_to = element['elementHeader'].get("effectiveTo", None)
+
+    # Get attributes from properties
+    # properties = element['properties']
+    # display_name = properties.get("name", "") or ""
+    # if display_name == "":
+    #     display_name = properties.get("displayName","")
+    # description = properties.get("description", "") or ""
+    # qualified_name = properties.get("qualifiedName", "") or ""
+    # category = properties.get("category", "") or ""
+    # version_identifier = properties.get("versionIdentifier", "") or ""
+    # additional_properties = properties.get("additionalProperties", {}) or {}
+    # extended_properties = properties.get("extendedProperties", {}) or {}
+    #
+    return {
+        "GUID": guid,
+        "metadata_collection_id": metadata_collection_id,
+        "metadata_collection_name": metadata_collection_name,
+        "origin_category": origin_category,
+        "created_by": created_by,
+        "create_time": create_time,
+        "updated_by": updated_by,
+        "version": version,
+        "type_name": type_name,
+        "classifications": classifications,
+
+        # "display_name": display_name,
+        # "description": description,
+        # "qualified_name": qualified_name,
+        # "category": category,
+        # "version_identifier": version_identifier,
+        # "additional_properties": additional_properties,
+        # "extended_properties": extended_properties,
+        "effective_from": effective_from,
+        "effective_to": effective_to,
+        }
+
+
+
+
+
+
+def markdown_to_html(markdown_text: str) -> str:
+    """
+    Convert markdown text to HTML, with special handling for mermaid code blocks.
+
+    Args:
+        markdown_text: The markdown text to convert
+
+    Returns:
+        HTML string
+    """
+    # Initialize markdown-it
+    md = MarkdownIt()
+
+    # Find all mermaid code blocks
+    mermaid_blocks = re.findall(r'```mermaid\n(.*?)\n```', markdown_text, re.DOTALL)
+
+    # Replace each mermaid block with a placeholder
+    placeholders = []
+    for i, block in enumerate(mermaid_blocks):
+        placeholder = f"MERMAID_PLACEHOLDER_{i}"
+        markdown_text = markdown_text.replace(f"```mermaid\n{block}\n```", placeholder)
+        placeholders.append((placeholder, block))
+
+    # Convert markdown to HTML
+    html_text = md.render(markdown_text)
+
+    # Replace placeholders with rendered mermaid HTML
+    for placeholder, mermaid_block in placeholders:
+        mermaid_html = construct_mermaid_web(mermaid_block)
+        html_text = html_text.replace(placeholder, mermaid_html)
+
+    # Add basic HTML structure
+    html_text = f"""
+    <!DOCTYPE html>
+    <html>
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Egeria Report</title>
+        <style>
+            body {{ font-family: Arial, sans-serif; line-height: 1.6; padding: 20px; }}
+            h1 {{ color: #2c3e50; }}
+            h2 {{ color: #3498db; }}
+            pre {{ background-color: #f8f8f8; padding: 10px; border-radius: 5px; overflow-x: auto; }}
+            table {{ border-collapse: collapse; width: 100%; margin-bottom: 20px; }}
+            th, td {{ border: 1px solid #ddd; padding: 8px; text-align: left; }}
+            th {{ background-color: #f2f2f2; }}
+            tr:nth-child(even) {{ background-color: #f9f9f9; }}
+        </style>
+    </head>
+    <body>
+        {html_text}
+    </body>
+    </html>
+    """
+
+    return html_text
+
+def make_preamble(obj_type: str, search_string: str, output_format: str = 'MD') -> Tuple[str, Optional[str]]:
+    """
+    Creates a preamble string and an elements action based on the given object type, search string,
+    and output format.
+
+    Args:
+        obj_type: The type of object being updated or reported on (e.g., "Product", "Category").
+        search_string: The search string used to filter objects. Defaults to "All Elements" if None.
+        output_format: A format identifier determining the output structure.
+            JSON - output standard json
+            MD - output standard markdown with no preamble
+            FORM - output markdown with a preamble for a form
+            REPORT - output markdown with a preamble for a report
+
+    Returns:
+        tuple: A tuple containing:
+            - A string representing the formatted update or report preamble.
+            - A string or None indicating the action description for the elements,
+              depending on the output format.
+    """
+    # search_string = search_string if search_string else "All Elements"
+    elements_md = ""
+    elements_action = "Update " + obj_type
+    if output_format == "FORM":
+        preamble = f"\n# Update {obj_type} Form - created at {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
+        if search_string:
+            preamble +=  f"\t {obj_type} found from the search string:  `{search_string}`\n\n"
+        return preamble, elements_action
+    elif output_format == "REPORT":
+        elements_md += (f"# {obj_type} Report - created at {datetime.now().strftime('%Y-%m-%d %H:%M')}\n"
+                       f"\t{obj_type}  found from the search string:  `{search_string}`\n\n")
+        elements_action = None
+        return elements_md, elements_action
+    else:
+        return "\n", elements_action
+
+def make_md_attribute(attribute_name: str, attribute_value: str, output_type: str) -> Optional[str]:
+    """
+    Create a markdown attribute line for a given attribute name and value.
+
+    Args:
+        attribute_name: The name of the attribute
+        attribute_value: The value of the attribute
+        output_type: The output format (FORM, MD, REPORT)
+
+    Returns:
+        str: Formatted markdown for the attribute
+    """
+    output = ""
+    if isinstance(attribute_value,str):
+        attribute_value = attribute_value.strip() if attribute_value else ""
+    elif isinstance(attribute_value,list) and len(attribute_value) > 0:
+        attribute_value = ",\n".join(attribute_value)
+    if attribute_name:
+        if attribute_name.upper() == "GUID":
+            attribute_title = attribute_name.upper()
+        else:
+            # attribute_title = attribute_name.title()
+            attribute_title = camel_to_title_case(attribute_name)
+    else:
+        attribute_title = ""
+
+    if output_type in ["FORM", "MD"]:
+        if attribute_name.lower() in [ "mermaid", "solutionBlueprintMermaidGraph", "links", "implemented by", "sub_components"]:
+            return '\n'
+
+        output = f"## {attribute_title}\n{attribute_value}\n\n"
+    elif output_type in ["REPORT", "MERMAID"]:
+        if attribute_title in ['Mermaid Graph', 'Mermaid', 'Solution Blueprint Mermaid Graph']:
+            output = f"## {attribute_title}\n\n```mermaid\n{attribute_value}\n```\n"
+        elif attribute_value:
+            output = f"## {attribute_title}\n{attribute_value}\n\n"
+    return output
+
+def format_for_markdown_table(text: str, guid: str = None) -> str:
+    """
+    Format text for markdown tables by replacing newlines with spaces and escaping pipe characters.
+    No truncation is applied to allow full-length text display regardless of console width.
+
+    Args:
+        text (str): The text to format
+
+    Returns:
+        str: Formatted text safe for markdown tables
+    """
+    if not text:
+        return ""
+    # Replace newlines with spaces and escape pipe characters
+    if isinstance(text, list):
+        text = "\n".join(text)
+    t = text.replace("\n", " ").replace("|", "\\|")
+    if '::' in t and guid:
+        t = f" [{t}](#{guid}) "
+    return t
+
+
+def populate_columns_from_properties(element: dict, columns_struct: dict) -> dict:
+    """
+    Populate a columns_struct with values from the element's properties.
+
+    The element dict is expected to have a nested 'properties' dict whose keys are in camelCase.
+    The columns_struct is expected to follow the format returned by select_report_spec, where
+    columns are located at columns_struct['formats']['columns'] and each column is a dict containing
+    at least a 'key' field expressed in snake_case. For each column whose snake_case key corresponds
+    to a key in the element properties (after converting to camelCase), this function adds a 'value'
+    entry to the column with the matching property's value.
+
+    Args:
+        element: The element containing a 'properties' dict with camelCase keys.
+        columns_struct: The columns structure whose columns have snake_case 'key' fields.
+
+    Returns:
+        The updated columns_struct (the input structure is modified in place and also returned).
+    """
+    if not isinstance(columns_struct, dict):
+        return columns_struct
+
+    props = (element or {}).get('properties') or {}
+    # If properties is not a dict, do nothing
+    if not isinstance(props, dict):
+        return columns_struct
+
+    # Get the attributes list if present
+    formats = columns_struct.get('formats') or {}
+    columns = formats.get('attributes') if isinstance(formats, dict) else None
+    if not isinstance(columns, list):
+        return columns_struct
+
+    for col in columns:
+        try:
+            key_snake = col.get('key') if isinstance(col, dict) else None
+            if not key_snake:
+                continue
+            # Convert the snake_case key to camelCase to look up in properties
+            key_camel = to_camel_case(key_snake)
+            if key_camel in props:
+                col['value'] = props.get(key_camel)
+        except Exception as e:
+            # Be resilient; log and continue
+            logger.debug(f"populate_columns_from_properties: skipping column due to error: {e}")
+            continue
+
+    return columns_struct
+
+
+def get_required_relationships(element: dict, columns_struct: dict) -> dict:
+    """
+    Populate relationship-derived column values in columns_struct based on top-level keys in the element.
+
+    This function inspects the requested columns in columns_struct, converts each column key from
+    snake_case to camelCase, and if a matching top-level key exists in the element, parses that value
+    (typically lists of relationship beans) into a human-readable value (e.g., a comma-separated list
+    of qualified names) and stores it under the column's 'value'. Columns not specified in the
+    columns_struct are ignored. Existing non-empty 'value's are left as-is.
+
+    Example: if a column with key 'member_of_collections' is present, this function will look for the
+    top-level key 'memberOfCollections' in the element and derive a value if found.
+
+    Args:
+        element: The element dictionary containing top-level relationship lists (e.g., associatedGlossaries,
+                 memberOfCollections, collectionMembers).
+        columns_struct: The columns structure to augment with derived 'value's.
+
+    Returns:
+        The updated columns_struct (modified in place and returned).
+    """
+    if not isinstance(columns_struct, dict):
+        return columns_struct
+
+    formats = columns_struct.get('formats') or {}
+    columns = formats.get('attributes') if isinstance(formats, dict) else None
+    if not isinstance(columns, list):
+        return columns_struct
+
+    def _extract_name_from_item(item: Any) -> Optional[str]:
+        """Best-effort extraction of a display/qualified name from a relationship item."""
+        try:
+            if isinstance(item, dict):
+                # Common pattern: item['relatedElement']['properties']['qualifiedName']
+                related = item.get('relatedElement') or item.get('related_element')
+                if isinstance(related, dict):
+                    props = related.get('properties') or {}
+                    name = (
+                        props.get('qualifiedName')
+                        or props.get('displayName')
+                        or props.get('name')
+                    )
+                    if name:
+                        return name
+                # Sometimes the properties may be at the top level of the item
+                name = (
+                    item.get('qualifiedName')
+                    or item.get('displayName')
+                    or item.get('name')
+                )
+                if name:
+                    return name
+            elif isinstance(item, str):
+                return item
+        except Exception as e:
+            logger.debug(f"get_required_relationships: error extracting name from item: {e}")
+        return None
+
+    for col in columns:
+        try:
+            if not isinstance(col, dict):
+                continue
+            key_snake = col.get('key')
+            if not key_snake:
+                continue
+            # If already has a non-empty value, don't overwrite
+            if col.get('value') not in (None, ""):
+                continue
+
+            # Convert the snake_case key to camelCase to look up in top-level element
+            key_camel = to_camel_case(key_snake)
+            if key_camel not in element:
+                continue
+
+            top_val = element.get(key_camel)
+            derived_value: str = ""
+            if isinstance(top_val, list):
+                names: List[str] = []
+                for item in top_val:
+                    nm = _extract_name_from_item(item)
+                    if nm:
+                        names.append(nm)
+                derived_value = ", ".join(names)
+            elif isinstance(top_val, dict):
+                nm = _extract_name_from_item(top_val)
+                derived_value = nm or ""
+            else:
+                # Primitive or unexpected type; coerce to string if not None
+                derived_value = str(top_val) if top_val is not None else ""
+
+            col['value'] = derived_value
+        except Exception as e:
+            logger.debug(f"get_required_relationships: skipping column due to error: {e}")
+            continue
+
+    return columns_struct
+
+
+def generate_entity_md(elements: List[Dict], 
+                      elements_action: str, 
+                      output_format: str, 
+                      entity_type: str,
+                      extract_properties_func: Callable, 
+                      get_additional_props_func: Optional[Callable] = None,
+                       columns_struct: [dict] = None) -> str:
+    """
+    Generic method to generate markdown for entities.
+
+    Args:
+        elements (list): List of entity elements
+        elements_action (str): Action description for elements
+        output_format (str): Output format
+        entity_type (str): Type of entity (Glossary, Term, Category, etc.)
+        extract_properties_func: Function to extract properties from an element
+        get_additional_props_func: Optional function to get additional properties
+        columns (list): List of column name structures
+
+    Returns:
+        str: Markdown representation
+    """
+    heading = columns_struct.get("heading")
+    if heading == "Default Base Attributes":
+        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of report_spec and output_format?\n\n"
+    else:
+        elements_md = ""
+    base_columns = columns_struct['formats'].get('attributes') if columns_struct else None
+
+    for element in elements:
+        if element is None:
+                continue
+        guid = element.get('elementHeader', {}).get('guid')
+
+        # Prefer new behavior: extractor returns an updated columns_struct with values
+        returned_struct = None
+        if columns_struct is not None:
+            try:
+                returned_struct = extract_properties_func(element, columns_struct)
+            except TypeError:
+                # Fallback for legacy extractors without columns_struct parameter
+                returned_struct = None
+        
+        # Legacy fallback: get props dict if no columns_struct provided/returned
+        props = {}
+        if returned_struct is None:
+            props = extract_properties_func(element) if callable(extract_properties_func) else {}
+
+        # Get additional properties if function is provided
+        additional_props = {}
+        if get_additional_props_func:
+            # Use guid if available, else try to get from props
+            guid_for_fmt = guid or props.get('GUID')
+            additional_props = get_additional_props_func(element, guid_for_fmt, output_format)
+
+        # Determine display name
+        display_name = None
+        if returned_struct is not None:
+            cols = returned_struct.get('formats', {}).get('attributes', [])
+            # Find value from 'display_name' or 'title'
+            for col in cols:
+                if col.get('key') in ('display_name', 'title'):
+                    display_name = col.get('value')
+                    if display_name:
+                        break
+        else:
+            display_name = props.get('display_name') or props.get('title')
+
+        if display_name is None:
+            display_name = "NO DISPLAY NAME"
+
+        # Format header based on output format
+        if output_format in ['FORM', 'MD']:
+            elements_md += f"# {elements_action}\n\n"
+            elements_md += f"## {entity_type} Name \n\n{display_name}\n\n"
+        elif output_format == 'REPORT':
+            elements_md += f'<a id="{(guid or props.get("GUID") or "No GUID" )}"></a>\n# {entity_type} Name: {display_name}\n\n'
+        else:
+            elements_md += f"## {entity_type} Name \n\n{display_name}\n\n"
+
+        # Add attributes based on column spec if available, otherwise, add all (legacy)
+        if returned_struct is not None:
+            cols = returned_struct.get('formats', {}).get('attributes', [])
+            for column in cols:
+                name = column.get('name')
+                key = column.get('key')
+                value = column.get('value')
+                if value in (None, "") and key in additional_props:
+                    value = additional_props[key]
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid)
+                elements_md += make_md_attribute(name, value, output_format)
+            if wk := returned_struct.get("annotations", {}).get("wikilinks", None):
+                elements_md += ", ".join(wk)
+        elif base_columns:
+            # If we have columns but extractor didn't return struct, use legacy props lookup
+            for column in base_columns:
+                key = column['key']
+                name = column['name']
+                value = ""
+                if key in props:
+                    value = props[key]
+                elif key in additional_props:
+                    value = additional_props[key]
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid or props.get('GUID'))
+                elements_md += make_md_attribute(name, value, output_format)
+            if wk := columns_struct.get("annotations", {}).get("wikilinks", None):
+                elements_md += ", ".join(wk)
+        else:
+            # Legacy path without columns: dump all props
+            for key, value in props.items():
+                if output_format in ['FORM', 'MD', 'DICT'] and key == 'mermaid':
+                    continue
+                if key not in ['properties', 'display_name']:
+                    if key == 'mermaid' and value == '':
+                        continue
+                    elements_md += make_md_attribute(key.replace('_', ' '), value, output_format)
+            for key, value in additional_props.items():
+                elements_md += make_md_attribute(key.replace('_', ' '), value, output_format)
+
+        if element != elements[-1]:
+            elements_md += MD_SEPARATOR
+
+    return elements_md
+
+def generate_entity_md_table(elements: List[Dict], 
+                            search_string: str, 
+                            entity_type: str, 
+                            extract_properties_func: Callable,
+                            columns_struct: dict,
+                            get_additional_props_func: Optional[Callable] = None, 
+                            output_format: str = 'LIST') -> str:
+    """
+    Generic method to generate a markdown table for entities.
+
+    Args:
+        elements (list): List of entity elements
+        search_string (str): The search string used
+        entity_type (str): Type of entity (Glossary, Term, Category, etc.)
+        extract_properties_func: Function to extract properties from an element
+        columns: List of column definitions, each containing 'name', 'key', and 'format' (optional)
+        get_additional_props_func: Optional function to get additional properties
+        output_format (str): Output format (FORM, REPORT, LIST, etc.)
+
+    Returns:
+        str: Markdown table
+    """
+    # Handle pluralization - if entity_type ends with 'y', use 'ies' instead of 's'
+    target_type = columns_struct.get('target_type', entity_type)
+    # if target_type.endswith('y'):
+    #     target_type = target_type.replace('y', 'ies')
+    # else:
+    #     target_type = target_type.replace('s', 's')
+
+    entity_type_plural = f"{target_type[:-1]}ies" if target_type.endswith('y') else f"{target_type}s"
+    # entity_type_plural = target_type
+    columns = columns_struct['formats'].get('attributes', [])
+    heading = columns_struct.get("heading")
+    if heading == "Default Base Attributes":
+        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of report_spec and output_format?\n\n"
+    else:
+        elements_md = ""
+
+    if output_format == "LIST":
+        elements_md = f"# {entity_type_plural} Table\n\n"
+        elements_md += f"{entity_type_plural} found from the search string: `{search_string}`\n\n"
+
+    # Add column headers
+    header_row = "| "
+    separator_row = "|"
+    for column in columns:
+        header_row += f"{column['name']} | "
+        separator_row += "-------------|"
+
+    elements_md += header_row + "\n"
+    elements_md += separator_row + "\n"
+
+    # Add rows
+    for element in elements:
+        guid = element.get('elementHeader', {}).get('guid', None)
+
+        # Extractor returns columns_struct with values when possible
+        try:
+            returned_struct = extract_properties_func(element, columns_struct)
+        except TypeError:
+            returned_struct = None
+
+        # For help mode, bypass extraction
+        if output_format == "help":
+            returned_struct = {"formats": {"attributes": columns}}
+
+        # Additional props (if any)
+        additional_props = {}
+        if get_additional_props_func:
+            additional_props = get_additional_props_func(element, guid, output_format)
+
+        # Build row
+        row = "| "
+        if returned_struct is not None:
+            for column in returned_struct.get('formats', {}).get('attributes', []):
+                key = column.get('key')
+                value = column.get('value')
+                if (value in (None, "")) and key in additional_props:
+                    value = additional_props[key]
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid)
+                row += f"{value} | "
+        else:
+            # Legacy fallback: read from props dict
+            props = extract_properties_func(element)
+            for column in columns:
+                key = column['key']
+                value = ""
+                if key in props:
+                    value = props[key]
+                elif key in additional_props:
+                    value = additional_props[key]
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid or props.get('GUID'))
+                row += f"{value} | "
+
+        elements_md += row + "\n"
+        # if wk := columns_struct.get("annotations",{}).get("wikilinks", None):
+        #     elements_md += ", ".join(wk)
+    return elements_md
+
+def generate_entity_dict(elements: List[Dict], 
+                        extract_properties_func: Callable, 
+                        get_additional_props_func: Optional[Callable] = None,
+                        include_keys: Optional[List[str]] = None, 
+                        exclude_keys: Optional[List[str]] = None,
+                        columns_struct: dict = None,
+                        output_format: str = 'DICT') -> List[Dict]:
+    """
+    Generic method to generate a dictionary representation of entities.
+
+    Args:
+        elements (list): List of entity elements
+        extract_properties_func: Function to extract properties from an element
+        get_additional_props_func: Optional function to get additional properties
+        include_keys: Optional list of keys to include in the result (if None, include all)
+        exclude_keys: Optional list of keys to exclude from the result (if None, exclude none)
+        columns_struct: Optional dict of columns to include (if None, include all)
+        output_format (str): Output format (FORM, REPORT, DICT, etc.)
+
+    Returns:
+        list: List of entity dictionaries
+    """
+    result = []
+
+    #####
+    # Add attributes based on column spec if available, otherwise, add all
+    for element in elements:
+        if element is None:
+            continue
+
+        guid = element.get('elementHeader', {}).get('guid')
+
+        returned_struct = None
+        if columns_struct is not None:
+            try:
+                returned_struct = extract_properties_func(element, columns_struct)
+            except TypeError as e:
+                logger.info(f"Error - didn't find extractor?: {e}")
+                returned_struct = None
+
+        # Get additional properties if function is provided
+        additional_props = {}
+        if get_additional_props_func:
+            additional_props = get_additional_props_func(element, guid, output_format)
+
+        # Create entity dictionary
+        entity_dict = {}
+
+        columns = columns_struct['formats'].get('attributes', None) if columns_struct else None
+        if returned_struct is not None:
+            for column in returned_struct.get('formats', {}).get('attributes', []):
+                key = column.get('key')
+                name = column.get('name')
+                value = column.get('value')
+                if (value in (None, "")) and key in additional_props:
+                    value = additional_props[key]
+                if column.get('format'):
+                    value = format_for_markdown_table(value, guid)
+                # Use machine key for DICT output to avoid name collisions (e.g., multiple 'GUID' display names)
+                dict_key = key or name
+                # If the key already exists and differs only by case, preserve both by preferring exact key
+                if dict_key in entity_dict and name and name != dict_key:
+                    # suffix with _1 to avoid overwrite
+                    suffix_idx = 1
+                    tmp_key = f"{dict_key}_{suffix_idx}"
+                    while tmp_key in entity_dict:
+                        suffix_idx += 1
+                        tmp_key = f"{dict_key}_{suffix_idx}"
+                    dict_key = tmp_key
+                entity_dict[dict_key] = value
+        elif columns:
+            for column in columns:
+                key = column['key']
+                name = column['name']
+                value = ""
+                props = extract_properties_func(element)
+                if key in props:
+                    value = props[key]
+                elif key in additional_props:
+                    value = additional_props[key]
+                if  column.get('format', None):
+                    value = format_for_markdown_table(value, guid or props.get('GUID'))
+                dict_key = key or name
+                if dict_key in entity_dict and name and name != dict_key:
+                    suffix_idx = 1
+                    tmp_key = f"{dict_key}_{suffix_idx}"
+                    while tmp_key in entity_dict:
+                        suffix_idx += 1
+                        tmp_key = f"{dict_key}_{suffix_idx}"
+                    dict_key = tmp_key
+                entity_dict[dict_key] = value
+        else:
+            props = extract_properties_func(element, columns_struct)
+            # Add properties based on include/exclude lists
+            for key, value in props.items():
+                if key not in ['properties', 'mermaid']:  # Skip the raw properties object
+                    if (include_keys is None or key in include_keys) and (
+                            exclude_keys is None or key not in exclude_keys):
+                        entity_dict[key] = value
+
+            # Add additional properties
+            for key, value in additional_props.items():
+                if (include_keys is None or key in include_keys) and (exclude_keys is None or key not in exclude_keys):
+                    entity_dict[key] = value
+
+        result.append(entity_dict)
+    #####
+    # for element in elements:
+    #     if element is None:
+    #         continue
+    #     props = extract_properties_func(element)
+    #
+    #     # Get additional properties if function is provided
+    #     additional_props = {}
+    #     if get_additional_props_func:
+    #         additional_props = get_additional_props_func(element,props['GUID'], output_format)
+    #
+    #     # Create entity dictionary
+    #     entity_dict = {}
+    #
+    #     # Add properties based on include/exclude lists
+    #     for key, value in props.items():
+    #         if key not in [ 'properties', 'mermaid']:  # Skip the raw properties object
+    #             if (include_keys is None or key in include_keys) and (
+    #                     exclude_keys is None or key not in exclude_keys):
+    #                 entity_dict[key] = value
+    #
+    #     # Add additional properties
+    #     for key, value in additional_props.items():
+    #         if (include_keys is None or key in include_keys) and (exclude_keys is None or key not in exclude_keys):
+    #             entity_dict[key] = value
+    #
+    #     result.append(entity_dict)
+
+    return result
+
+def resolve_output_formats(entity_type: str,
+                           output_format: str,
+                           report_spec: Optional[Union[str, dict]] = None,
+                           default_label: Optional[str] = None,
+                           **kwargs) -> Optional[dict]:
+    """
+    Resolve a report format structure given an entity type, the desired output format
+    (e.g., DICT, LIST, MD, REPORT, FORM), and either a label (str) or a dict of format sets.
+
+    Backward compatibility:
+    - Accepts legacy kwarg 'report_spec' and treats it as report_spec.
+
+    Selection order:
+    - If report_spec is a str: select by label.
+    - If report_spec is a dict: use get_report_spec_match to pick a matching format.
+    - Else: try selecting by entity_type or default_label.
+    - Fallback: select "Default".
+    """
+    from pyegeria.base_report_formats import get_report_spec_match
+
+    if report_spec is None and isinstance(kwargs, dict):
+        if 'report_spec' in kwargs:
+            report_spec = kwargs.get('report_spec')
+        elif 'report_format' in kwargs:
+            report_spec = kwargs.get('report_format')
+        elif 'output_format_spec' in kwargs:
+            report_spec = kwargs.get('output_format_spec')
+
+    if isinstance(report_spec, str):
+        return select_report_format(report_spec, output_format)
+    if isinstance(report_spec, dict):
+        return get_report_spec_match(report_spec, output_format)
+
+    label = default_label or entity_type
+    fmt = select_report_format(label, output_format)
+    if fmt is None:
+        fmt = select_report_format("Default", output_format)
+    return fmt
+
+
+def overlay_additional_values(columns_struct: dict, extra: Optional[dict]) -> dict:
+    """
+    Overlay extra values into columns_struct only where the column's value is empty or missing.
+    Returns the modified columns_struct.
+    """
+    if not isinstance(columns_struct, dict) or not extra:
+        return columns_struct
+    columns = columns_struct.get('formats', {}).get('attributes')
+    if not isinstance(columns, list):
+        return columns_struct
+    for col in columns:
+        if not isinstance(col, dict):
+            continue
+        key = col.get('key')
+        if not key:
+            continue
+        if col.get('value') in (None, "") and key in extra:
+            col['value'] = extra[key]
+    return columns_struct
+
+
+def populate_common_columns(
+    element: dict,
+    columns_struct: dict,
+    *,
+    include_header: bool = True,
+    include_relationships: bool = True,
+    include_subject_area: bool = True,
+    mermaid_source_key: str = 'mermaidGraph',
+    mermaid_dest_key: str = 'mermaid'
+) -> dict:
+    """
+    Populate the common columns in columns_struct based on a standard Egeria element shape.
+
+    Steps:
+    - Populate from element.properties (camelCase mapped from snake_case keys)
+    - Optionally overlay header-derived values (GUID, type_name, times, etc.)
+    - Optionally populate relationship-based columns via get_required_relationships
+    - Optionally populate subject_area from element.elementHeader.subjectArea.classificationProperties.subjectAreaName
+    - If a column with key == mermaid_dest_key is present, set it from mermaid_source_key
+    - Do not overwrite non-empty values already set
+    """
+    # 1) Base properties
+    col_data = populate_columns_from_properties(element, columns_struct)
+    columns_list = col_data.get('formats', {}).get('attributes', [])
+
+    # 2) Header overlay
+    header_props = _extract_referenceable_properties(element) if include_header else {}
+    guid = header_props.get('GUID') if include_header else None
+    if include_header:
+        for column in columns_list:
+            if not isinstance(column, dict):
+                continue
+            key = column.get('key')
+            if not key:
+                continue
+            if column.get('value') not in (None, ""):
+                continue
+            if key in header_props:
+                column['value'] = header_props.get(key)
+            elif isinstance(key, str) and key.lower() == 'guid':
+                column['value'] = guid
+
+    # 3) Relationships
+    if include_relationships:
+        col_data = get_required_relationships(element, col_data)
+
+    # 4) Subject area
+    if include_subject_area:
+        try:
+            subject_area = element.get('elementHeader', {}).get('subjectArea') or ""
+            subj_val = ""
+            if isinstance(subject_area, dict):
+                subj_val = subject_area.get('classificationProperties', {}).get('subjectAreaName', '')
+            for column in columns_list:
+                if column.get('key') == 'subject_area' and column.get('value') in (None, ""):
+                    column['value'] = subj_val
+        except Exception as e:
+            logger.debug(f"populate_common_columns: subject_area handling error: {e}")
+
+    # 5) Mermaid
+    try:
+        mermaid_val = element.get(mermaid_source_key, '') or ''
+        for column in columns_list:
+            if column.get('key') == mermaid_dest_key and column.get('value') in (None, ""):
+                column['value'] = mermaid_val
+                break
+    except Exception as e:
+        logger.debug(f"populate_common_columns: mermaid handling error: {e}")
+
+    return col_data
+
+
+def extract_mermaid_only(elements: Union[Dict, List[Dict]]) -> Union[str, List[str]]:
+    """
+    Extract mermaid graph data from elements.
+
+    Args:
+        elements: Dictionary or list of dictionaries containing element data
+
+    Returns:
+        String or list of strings containing mermaid graph data
+    """
+    if isinstance(elements, dict):
+        mer = elements.get('mermaidGraph', None)
+        if mer:
+            return f"\n```mermaid\n{mer}\n```"
+        else:
+            return "---"
+
+
+    result = []
+    for element in elements:
+        mer = element.get('mermaidGraph', "---")
+        mer_out = f"\n\n```mermaid\n{mer}\n```" if mer else "---"
+        result.append(mer_out)
+    return result
+
+def extract_basic_dict(elements: Union[Dict, List[Dict]]) -> Union[Dict, List[Dict]]:
+    """
+    Extract basic dictionary data from elements.
+
+    Args:
+        elements: Dictionary or list of dictionaries containing element data
+
+    Returns:
+        Dictionary or list of dictionaries with extracted data
+    """
+    if isinstance(elements, dict):
+        body = {'guid': elements['elementHeader']['guid']}
+        for key in elements['properties']:
+            body[key] = elements['properties'][key]
+
+        # Add classifications if present
+        classifications = elements['elementHeader'].get('classifications', [])
+        if classifications:
+            classification_names = "["
+            for classification in classifications:
+                if len(classification_names) > 1:
+                    classification_names += ", "
+                classification_names += f"{classification['classificationName']}"
+            body['classification_names'] = classification_names + ']'
+
+        return body
+
+    result = []
+    for element in elements:
+        if element is None:
+            continue
+        body = {'guid': element['elementHeader']['guid']}
+        for key in element['properties']:
+            body[key] = element['properties'][key]
+
+        # Add classifications if present
+        classifications = element['elementHeader'].get('classifications', [])
+        if classifications:
+            classification_names = "["
+            for classification in classifications:
+                if len(classification_names) > 1:
+                    classification_names += ", "
+                classification_names += f"{classification['classificationName']}"
+            body['classifications'] = classification_names + ']'
+
+        result.append(body)
+    return result
+
+def _extract_default_properties(self, element: dict, columns_struct: dict) -> dict:
+    props = element.get('properties', {}) or {}
+    normalized = {
+        'properties': props,
+        'elementHeader': element.get('elementHeader', {}),
+    }
+    # Common population pipeline
+    col_data = populate_common_columns(element, columns_struct)
+    columns_list = col_data.get('formats', {}).get('attributes', [])
+
+    return col_data
+
+
+def _generate_default_output(self, elements: dict | list[dict], search_string: str,
+                             element_type_name: str | None,
+                             output_format: str = 'DICT',
+                             report_format: dict | str | None = None,
+                             **kwargs) -> str | list[dict]:
+    entity_type = 'Referenceable'
+    # Backward compatibility: accept legacy kwarg
+    if report_format is None and isinstance(kwargs, dict) and 'report_spec' in kwargs:
+        report_format = kwargs.get('report_spec')
+    if report_format:
+        if isinstance(report_format, str):
+            output_formats = select_report_format(report_format, output_format)
+        elif isinstance(report_format, dict):
+            output_formats = get_report_spec_match(report_format, output_format)
+        else:
+            output_formats = None
+    else:
+        output_formats = select_report_format(entity_type, output_format)
+    if output_formats is None:
+        output_formats = select_report_format('Default', output_format)
+    return generate_output(
+        elements=elements,
+        search_string=search_string,
+        entity_type=entity_type,
+        output_format=output_format,
+        extract_properties_func=_extract_default_properties,
+        get_additional_props_func=None,
+        columns_struct=output_formats,
+    )
+
+
+def generate_output(elements: Union[Dict, List[Dict]],
+               search_string: str,
+               entity_type: str,
+               output_format: str,
+               extract_properties_func: Callable,
+               get_additional_props_func: Optional[Callable] = None,
+               columns_struct: dict = None) -> Union[str, list[dict]]:
+    """
+    Generate output in the specified format for the given elements.
+
+    Args:
+        elements: Dictionary or list of dictionaries containing element data
+        search_string: The search string used to find the elements
+        entity_type: The type of entity (e.g., "Glossary", "Term", "Category")
+        output_format: The desired output format (MD, FORM, REPORT, LIST, DICT, MERMAID, HTML)
+        extract_properties_func: Function to extract properties from an element
+        get_additional_props_func: Optional function to get additional properties
+        columns: Optional list of column definitions for table output
+
+    Returns:
+        Formatted output as string or list of dictionaries
+    """
+    columns = columns_struct['formats'].get('attributes',None) if columns_struct else None
+    if not columns:
+        columns_struct = select_report_format("Default",output_format)
+        if columns_struct:
+            columns = columns_struct.get('formats', {}).get('attributes', None)
+
+    target_type = columns_struct.get('target_type', entity_type) if columns_struct else entity_type
+    if target_type is None:
+        target_type = entity_type
+
+    # Ensure elements is a list
+    if isinstance(elements, dict):
+        elements = [elements]
+
+    # Handle empty search string
+    if search_string is None or search_string == '':
+        search_string = "All"
+
+    # Set the output format to DICT to return values to table display
+    if output_format == "TABLE":
+        output_format = "DICT"
+
+    # Generate output based on format
+    if output_format == 'MERMAID':
+        return extract_mermaid_only(elements)
+
+    elif output_format == 'HTML':
+        # First generate the REPORT format output
+        report_output = generate_output(
+            elements=elements,
+            search_string=search_string,
+            entity_type=entity_type,
+            output_format="REPORT",
+            extract_properties_func=extract_properties_func,
+            get_additional_props_func=get_additional_props_func,
+            columns_struct=columns_struct
+        )
+
+        # Convert the markdown to HTML
+        return markdown_to_html(report_output)
+
+    elif output_format == 'DICT':
+        return generate_entity_dict(
+            elements=elements,
+            extract_properties_func=extract_properties_func,
+            get_additional_props_func=get_additional_props_func,
+            exclude_keys=['properties'],
+            columns_struct=columns_struct,
+            output_format=output_format
+        )
+
+    elif output_format == 'LIST':
+        if columns is None:
+            raise ValueError("Columns must be provided for LIST output format")
+
+        return generate_entity_md_table(
+            elements=elements,
+            search_string=search_string,
+            entity_type=entity_type,
+            extract_properties_func=extract_properties_func,
+            columns_struct=columns_struct,
+            get_additional_props_func=get_additional_props_func,
+            output_format=output_format
+        )
+
+    else:  #  MD, FORM, REPORT
+        elements_md, elements_action = make_preamble(
+            obj_type=target_type,
+            search_string=search_string,
+            output_format=output_format
+        )
+
+        elements_md += generate_entity_md(
+            elements=elements,
+            elements_action=elements_action,
+            output_format=output_format,
+            entity_type=target_type,
+            extract_properties_func=extract_properties_func,
+            get_additional_props_func=get_additional_props_func,
+            columns_struct = columns_struct
+        )
+
+        return elements_md
+
+# Machine keys explanation
+# -------------------------
+# In pyegeria’s reporting system, each output column is defined with two identifiers:
+# - name: the human-readable display label (e.g., "GUID", "Display Name") shown in tables and reports.
+# - key: the stable, programmatic identifier (usually snake_case, e.g., guid, display_name) used internally.
+#
+# We refer to the key as the machine key. It is intended to be:
+# - Unique within a given row/spec: so downstream code can unambiguously reference values.
+# - Stable over time: it should not change with presentation tweaks or localization.
+# - Program-friendly: lowercase with underscores, matching the data model or derivation (e.g., header vs properties).
+#
+# Why DICT output uses machine keys
+# - Display names are for humans and may repeat across different sources of data in one row. For example,
+#   Collections may include a GUID from the element header (key: guid, display name: "GUID") and a related member GUID
+#   (key: GUID, display name: "GUID"). If the DICT used display names, one would overwrite the other.
+# - By using the machine key (column['key']) as the dictionary key, DICT output avoids these collisions and remains
+#   deterministic for programmatic consumers.
+#
+# Collision handling
+# - If, within an unusual spec, two columns share the same key, DICT generation adds a numeric suffix (e.g., guid_1)
+#   to preserve both values. This is a safeguard and such collisions should be avoided in specs.
+#
+# Guidance for spec authors
+# - Choose clear, stable machine keys in snake_case that map to actual data fields or well-defined derivations.
+# - Reserve display names for presentation; do not rely on them for programmatic access.
+# - When you need two visually identical labels (e.g., both "GUID"), ensure their keys differ (e.g., guid vs GUID) to
+#   reflect different sources/semantics.