pyegeria 5.4.0.28__py3-none-any.whl → 5.5.3.3__py3-none-any.whl

pyegeria/{output_formatter.py → deprecated/output_formatter_with_machine_keys.py}

@@ -1,28 +1,31 @@
 from datetime import datetime
 import re
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
-from pyegeria.utils import (camel_to_title_case)
+from pyegeria.core.utils import (camel_to_title_case)
 from markdown_it import MarkdownIt
 from rich.console import Console
 from loguru import logger
+from pyegeria.core.config import settings
 
-from pyegeria.mermaid_utilities import construct_mermaid_web
-from pyegeria._output_formats import select_output_format_set, MD_SEPARATOR
+from pyegeria.view.mermaid_utilities import construct_mermaid_web
+from pyegeria.view.base_report_formats import select_report_format, MD_SEPARATOR, get_report_spec_match
 from pyegeria.models import to_camel_case
 
 """
-Note on select_output_format_set function:
+Note on select_report_spec function:
 
 This function and related data structures have been moved back to _output_formats.py.
-Please import select_output_format_set from pyegeria._output_formats instead of from this module.
+Please import select_report_spec from pyegeria._output_formats instead of from this module.
 """
 
-console = Console(width= 250)
+console = Console(width=settings.Environment.console_width)
 
 
 def _extract_referenceable_properties(element: dict[str, Any]) -> dict[str, Any]:
     # Get general header attributes
-    guid = element['elementHeader'].get("guid", None)
+    guid = element.get('elementHeader', {}).get("guid", None)
+    if guid is None:
+        return {}
     metadata_collection_id = element['elementHeader']['origin'].get("homeMetadataCollectionId", None)
     metadata_collection_name = element['elementHeader']['origin'].get("homeMetadataCollectionName", None)
     origin_category = element['elementHeader'].get("origin_category", None)
@@ -196,13 +199,13 @@ def make_md_attribute(attribute_name: str, attribute_value: str, output_type: st
         attribute_title = ""
 
     if output_type in ["FORM", "MD"]:
-        if attribute_name.lower() in [ "mermaid", "links", "implemented by", "sub_components"]:
+        if attribute_name.lower() in [ "mermaid", "solutionBlueprintMermaidGraph", "links", "implemented by", "sub_components"]:
             return '\n'
 
         output = f"## {attribute_title}\n{attribute_value}\n\n"
-    elif output_type == "REPORT":
-        if attribute_title in ['Mermaid Graph', 'Mermaid']:
-            output = f"## Mermaid Graph\n\n```mermaid\n{attribute_value}\n```\n"
+    elif output_type in ["REPORT", "MERMAID"]:
+        if attribute_title in ['Mermaid Graph', 'Mermaid', 'Solution Blueprint Mermaid Graph']:
+            output = f"## {attribute_title}\n\n```mermaid\n{attribute_value}\n```\n"
         elif attribute_value:
             output = f"## {attribute_title}\n{attribute_value}\n\n"
     return output
@@ -234,7 +237,7 @@ def populate_columns_from_properties(element: dict, columns_struct: dict) -> dic
     Populate a columns_struct with values from the element's properties.
 
     The element dict is expected to have a nested 'properties' dict whose keys are in camelCase.
-    The columns_struct is expected to follow the format returned by select_output_format_set, where
+    The columns_struct is expected to follow the format returned by select_report_spec, where
     columns are located at columns_struct['formats']['columns'] and each column is a dict containing
     at least a 'key' field expressed in snake_case. For each column whose snake_case key corresponds
     to a key in the element properties (after converting to camelCase), this function adds a 'value'
@@ -255,9 +258,9 @@ def populate_columns_from_properties(element: dict, columns_struct: dict) -> dic
     if not isinstance(props, dict):
         return columns_struct
 
-    # Get the columns list if present
+    # Get the attributes list if present
     formats = columns_struct.get('formats') or {}
-    columns = formats.get('columns') if isinstance(formats, dict) else None
+    columns = formats.get('attributes') if isinstance(formats, dict) else None
     if not isinstance(columns, list):
         return columns_struct
 
@@ -303,7 +306,7 @@ def get_required_relationships(element: dict, columns_struct: dict) -> dict:
         return columns_struct
 
     formats = columns_struct.get('formats') or {}
-    columns = formats.get('columns') if isinstance(formats, dict) else None
+    columns = formats.get('attributes') if isinstance(formats, dict) else None
     if not isinstance(columns, list):
         return columns_struct
 
@@ -400,10 +403,10 @@ def generate_entity_md(elements: List[Dict],
     """
     heading = columns_struct.get("heading")
     if heading == "Default Base Attributes":
-        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of output_format_set and output_format?\n\n"
+        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of report_spec and output_format?\n\n"
     else:
         elements_md = ""
-    base_columns = columns_struct['formats'].get('columns') if columns_struct else None
+    base_columns = columns_struct['formats'].get('attributes') if columns_struct else None
 
     for element in elements:
         if element is None:
@@ -434,7 +437,7 @@ def generate_entity_md(elements: List[Dict],
         # Determine display name
         display_name = None
         if returned_struct is not None:
-            cols = returned_struct.get('formats', {}).get('columns', [])
+            cols = returned_struct.get('formats', {}).get('attributes', [])
             # Find value from 'display_name' or 'title'
             for col in cols:
                 if col.get('key') in ('display_name', 'title'):
@@ -458,7 +461,7 @@ def generate_entity_md(elements: List[Dict],
 
         # Add attributes based on column spec if available, otherwise, add all (legacy)
         if returned_struct is not None:
-            cols = returned_struct.get('formats', {}).get('columns', [])
+            cols = returned_struct.get('formats', {}).get('attributes', [])
             for column in cols:
                 name = column.get('name')
                 key = column.get('key')
@@ -468,7 +471,7 @@ def generate_entity_md(elements: List[Dict],
                 if column.get('format'):
                     value = format_for_markdown_table(value, guid)
                 elements_md += make_md_attribute(name, value, output_format)
-            if wk := returned_struct.get("annotations", {}).get("wikilinks"):
+            if wk := returned_struct.get("annotations", {}).get("wikilinks", None):
                 elements_md += ", ".join(wk)
         elif base_columns:
             # If we have columns but extractor didn't return struct, use legacy props lookup
@@ -525,11 +528,18 @@ def generate_entity_md_table(elements: List[Dict],
         str: Markdown table
     """
     # Handle pluralization - if entity_type ends with 'y', use 'ies' instead of 's'
-    entity_type_plural = f"{entity_type[:-1]}ies" if entity_type.endswith('y') else f"{entity_type}s"
-    columns = columns_struct['formats'].get('columns', [])
+    target_type = columns_struct.get('target_type', entity_type)
+    # if target_type.endswith('y'):
+    #     target_type = target_type.replace('y', 'ies')
+    # else:
+    #     target_type = target_type.replace('s', 's')
+
+    entity_type_plural = f"{target_type[:-1]}ies" if target_type.endswith('y') else f"{target_type}s"
+    # entity_type_plural = target_type
+    columns = columns_struct['formats'].get('attributes', [])
     heading = columns_struct.get("heading")
     if heading == "Default Base Attributes":
-        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of output_format_set and output_format?\n\n"
+        elements_md = "## Reporting on Default Base Attributes - Perhaps couldn't find a valid combination of report_spec and output_format?\n\n"
     else:
         elements_md = ""
 
@@ -549,7 +559,7 @@ def generate_entity_md_table(elements: List[Dict],
 
     # Add rows
     for element in elements:
-        guid = element.get('elementHeader', {}).get('guid')
+        guid = element.get('elementHeader', {}).get('guid', None)
 
         # Extractor returns columns_struct with values when possible
         try:
@@ -559,7 +569,7 @@ def generate_entity_md_table(elements: List[Dict],
 
         # For help mode, bypass extraction
         if output_format == "help":
-            returned_struct = {"formats": {"columns": columns}}
+            returned_struct = {"formats": {"attributes": columns}}
 
         # Additional props (if any)
         additional_props = {}
@@ -569,7 +579,7 @@ def generate_entity_md_table(elements: List[Dict],
         # Build row
         row = "| "
         if returned_struct is not None:
-            for column in returned_struct.get('formats', {}).get('columns', []):
+            for column in returned_struct.get('formats', {}).get('attributes', []):
                 key = column.get('key')
                 value = column.get('value')
                 if (value in (None, "")) and key in additional_props:
@@ -592,8 +602,8 @@ def generate_entity_md_table(elements: List[Dict],
                 row += f"{value} | "
 
         elements_md += row + "\n"
-        if wk := columns_struct.get("annotations",{}).get("wikilinks", None):
-            elements_md += ", ".join(wk)
+        # if wk := columns_struct.get("annotations",{}).get("wikilinks", None):
+        #     elements_md += ", ".join(wk)
     return elements_md
 
 def generate_entity_dict(elements: List[Dict], 
@@ -644,9 +654,9 @@ def generate_entity_dict(elements: List[Dict],
         # Create entity dictionary
         entity_dict = {}
 
-        columns = columns_struct['formats'].get('columns', None) if columns_struct else None
+        columns = columns_struct['formats'].get('attributes', None) if columns_struct else None
         if returned_struct is not None:
-            for column in returned_struct.get('formats', {}).get('columns', []):
+            for column in returned_struct.get('formats', {}).get('attributes', []):
                 key = column.get('key')
                 name = column.get('name')
                 value = column.get('value')
@@ -654,7 +664,18 @@ def generate_entity_dict(elements: List[Dict],
                     value = additional_props[key]
                 if column.get('format'):
                     value = format_for_markdown_table(value, guid)
-                entity_dict[name] = value
+                # Use machine key for DICT output to avoid name collisions (e.g., multiple 'GUID' display names)
+                dict_key = key or name
+                # If the key already exists and differs only by case, preserve both by preferring exact key
+                if dict_key in entity_dict and name and name != dict_key:
+                    # suffix with _1 to avoid overwrite
+                    suffix_idx = 1
+                    tmp_key = f"{dict_key}_{suffix_idx}"
+                    while tmp_key in entity_dict:
+                        suffix_idx += 1
+                        tmp_key = f"{dict_key}_{suffix_idx}"
+                    dict_key = tmp_key
+                entity_dict[dict_key] = value
         elif columns:
             for column in columns:
                 key = column['key']
@@ -667,9 +688,17 @@ def generate_entity_dict(elements: List[Dict],
                     value = additional_props[key]
                 if  column.get('format', None):
                     value = format_for_markdown_table(value, guid or props.get('GUID'))
-                entity_dict[name] = value
+                dict_key = key or name
+                if dict_key in entity_dict and name and name != dict_key:
+                    suffix_idx = 1
+                    tmp_key = f"{dict_key}_{suffix_idx}"
+                    while tmp_key in entity_dict:
+                        suffix_idx += 1
+                        tmp_key = f"{dict_key}_{suffix_idx}"
+                    dict_key = tmp_key
+                entity_dict[dict_key] = value
         else:
-            props = extract_properties_func(element)
+            props = extract_properties_func(element, columns_struct)
             # Add properties based on include/exclude lists
             for key, value in props.items():
                 if key not in ['properties', 'mermaid']:  # Skip the raw properties object
@@ -713,6 +742,139 @@ def generate_entity_dict(elements: List[Dict],
 
     return result
 
+def resolve_output_formats(entity_type: str,
+                           output_format: str,
+                           report_spec: Optional[Union[str, dict]] = None,
+                           default_label: Optional[str] = None,
+                           **kwargs) -> Optional[dict]:
+    """
+    Resolve a report format structure given an entity type, the desired output format
+    (e.g., DICT, LIST, MD, REPORT, FORM), and either a label (str) or a dict of format sets.
+
+    Backward compatibility:
+    - Accepts legacy kwarg 'report_spec' and treats it as report_spec.
+
+    Selection order:
+    - If report_spec is a str: select by label.
+    - If report_spec is a dict: use get_report_spec_match to pick a matching format.
+    - Else: try selecting by entity_type or default_label.
+    - Fallback: select "Default".
+    """
+    from pyegeria.view.base_report_formats import get_report_spec_match
+
+    if report_spec is None and isinstance(kwargs, dict):
+        if 'report_spec' in kwargs:
+            report_spec = kwargs.get('report_spec')
+        elif 'report_format' in kwargs:
+            report_spec = kwargs.get('report_format')
+        elif 'output_format_spec' in kwargs:
+            report_spec = kwargs.get('output_format_spec')
+
+    if isinstance(report_spec, str):
+        return select_report_format(report_spec, output_format)
+    if isinstance(report_spec, dict):
+        return get_report_spec_match(report_spec, output_format)
+
+    label = default_label or entity_type
+    fmt = select_report_format(label, output_format)
+    if fmt is None:
+        fmt = select_report_format("Default", output_format)
+    return fmt
+
+
+def overlay_additional_values(columns_struct: dict, extra: Optional[dict]) -> dict:
+    """
+    Overlay extra values into columns_struct only where the column's value is empty or missing.
+    Returns the modified columns_struct.
+    """
+    if not isinstance(columns_struct, dict) or not extra:
+        return columns_struct
+    columns = columns_struct.get('formats', {}).get('attributes')
+    if not isinstance(columns, list):
+        return columns_struct
+    for col in columns:
+        if not isinstance(col, dict):
+            continue
+        key = col.get('key')
+        if not key:
+            continue
+        if col.get('value') in (None, "") and key in extra:
+            col['value'] = extra[key]
+    return columns_struct
+
+
+def populate_common_columns(
+    element: dict,
+    columns_struct: dict,
+    *,
+    include_header: bool = True,
+    include_relationships: bool = True,
+    include_subject_area: bool = True,
+    mermaid_source_key: str = 'mermaidGraph',
+    mermaid_dest_key: str = 'mermaid'
+) -> dict:
+    """
+    Populate the common columns in columns_struct based on a standard Egeria element shape.
+
+    Steps:
+    - Populate from element.properties (camelCase mapped from snake_case keys)
+    - Optionally overlay header-derived values (GUID, type_name, times, etc.)
+    - Optionally populate relationship-based columns via get_required_relationships
+    - Optionally populate subject_area from element.elementHeader.subjectArea.classificationProperties.subjectAreaName
+    - If a column with key == mermaid_dest_key is present, set it from mermaid_source_key
+    - Do not overwrite non-empty values already set
+    """
+    # 1) Base properties
+    col_data = populate_columns_from_properties(element, columns_struct)
+    columns_list = col_data.get('formats', {}).get('attributes', [])
+
+    # 2) Header overlay
+    header_props = _extract_referenceable_properties(element) if include_header else {}
+    guid = header_props.get('GUID') if include_header else None
+    if include_header:
+        for column in columns_list:
+            if not isinstance(column, dict):
+                continue
+            key = column.get('key')
+            if not key:
+                continue
+            if column.get('value') not in (None, ""):
+                continue
+            if key in header_props:
+                column['value'] = header_props.get(key)
+            elif isinstance(key, str) and key.lower() == 'guid':
+                column['value'] = guid
+
+    # 3) Relationships
+    if include_relationships:
+        col_data = get_required_relationships(element, col_data)
+
+    # 4) Subject area
+    if include_subject_area:
+        try:
+            subject_area = element.get('elementHeader', {}).get('subjectArea') or ""
+            subj_val = ""
+            if isinstance(subject_area, dict):
+                subj_val = subject_area.get('classificationProperties', {}).get('subjectAreaName', '')
+            for column in columns_list:
+                if column.get('key') == 'subject_area' and column.get('value') in (None, ""):
+                    column['value'] = subj_val
+        except Exception as e:
+            logger.debug(f"populate_common_columns: subject_area handling error: {e}")
+
+    # 5) Mermaid
+    try:
+        mermaid_val = element.get(mermaid_source_key, '') or ''
+        for column in columns_list:
+            if column.get('key') == mermaid_dest_key and column.get('value') in (None, ""):
+                column['value'] = mermaid_val
+                break
+    except Exception as e:
+        logger.debug(f"populate_common_columns: mermaid handling error: {e}")
+
+    return col_data
+
+
 def extract_mermaid_only(elements: Union[Dict, List[Dict]]) -> Union[str, List[str]]:
     """
     Extract mermaid graph data from elements.
@@ -724,11 +886,18 @@ def extract_mermaid_only(elements: Union[Dict, List[Dict]]) -> Union[str, List[s
         String or list of strings containing mermaid graph data
     """
     if isinstance(elements, dict):
-        return elements.get('mermaidGraph', '___')
+        mer = elements.get('mermaidGraph', None)
+        if mer:
+            return f"\n```mermaid\n{mer}\n```"
+        else:
+            return "---"
+
 
     result = []
     for element in elements:
-        result.append(element.get('mermaidGraph', '___'))
+        mer = element.get('mermaidGraph', "---")
+        mer_out = f"\n\n```mermaid\n{mer}\n```" if mer else "---"
+        result.append(mer_out)
     return result
 
 def extract_basic_dict(elements: Union[Dict, List[Dict]]) -> Union[Dict, List[Dict]]:
@@ -779,13 +948,57 @@ def extract_basic_dict(elements: Union[Dict, List[Dict]]) -> Union[Dict, List[Di
         result.append(body)
     return result
 
-def generate_output(elements: Union[Dict, List[Dict]], 
-                   search_string: str,
-                   entity_type: str,
-                   output_format: str,
-                   extract_properties_func: Callable,
-                   get_additional_props_func: Optional[Callable] = None,
-                   columns_struct: dict = None) -> Union[str, list[dict]]:
+def _extract_default_properties(self, element: dict, columns_struct: dict) -> dict:
+    props = element.get('properties', {}) or {}
+    normalized = {
+        'properties': props,
+        'elementHeader': element.get('elementHeader', {}),
+    }
+    # Common population pipeline
+    col_data = populate_common_columns(element, columns_struct)
+    columns_list = col_data.get('formats', {}).get('attributes', [])
+
+    return col_data
+
+
+def _generate_default_output(self, elements: dict | list[dict], search_string: str,
+                             element_type_name: str | None,
+                             output_format: str = 'DICT',
+                             report_format: dict | str | None = None,
+                             **kwargs) -> str | list[dict]:
+    entity_type = 'Referenceable'
+    # Backward compatibility: accept legacy kwarg
+    if report_format is None and isinstance(kwargs, dict) and 'report_spec' in kwargs:
+        report_format = kwargs.get('report_spec')
+    if report_format:
+        if isinstance(report_format, str):
+            output_formats = select_report_format(report_format, output_format)
+        elif isinstance(report_format, dict):
+            output_formats = get_report_spec_match(report_format, output_format)
+        else:
+            output_formats = None
+    else:
+        output_formats = select_report_format(entity_type, output_format)
+    if output_formats is None:
+        output_formats = select_report_format('Default', output_format)
+    return generate_output(
+        elements=elements,
+        search_string=search_string,
+        entity_type=entity_type,
+        output_format=output_format,
+        extract_properties_func=_extract_default_properties,
+        get_additional_props_func=None,
+        columns_struct=output_formats,
+    )
+
+
+def generate_output(elements: Union[Dict, List[Dict]],
+               search_string: str,
+               entity_type: str,
+               output_format: str,
+               extract_properties_func: Callable,
+               get_additional_props_func: Optional[Callable] = None,
+               columns_struct: dict = None) -> Union[str, list[dict]]:
     """
     Generate output in the specified format for the given elements.
 
@@ -801,7 +1014,15 @@ def generate_output(elements: Union[Dict, List[Dict]],
     Returns:
         Formatted output as string or list of dictionaries
     """
-    columns = columns_struct['formats'].get('columns',None) if columns_struct else None
+    columns = columns_struct['formats'].get('attributes',None) if columns_struct else None
+    if not columns:
+        columns_struct = select_report_format("Default",output_format)
+        if columns_struct:
+            columns = columns_struct.get('formats', {}).get('attributes', None)
+
+    target_type = columns_struct.get('target_type', entity_type) if columns_struct else entity_type
+    if target_type is None:
+        target_type = entity_type
 
     # Ensure elements is a list
     if isinstance(elements, dict):
@@ -811,6 +1032,10 @@ def generate_output(elements: Union[Dict, List[Dict]],
     if search_string is None or search_string == '':
         search_string = "All"
 
+    # Set the output format to DICT to return values to table display
+    if output_format == "TABLE":
+        output_format = "DICT"
+
     # Generate output based on format
     if output_format == 'MERMAID':
         return extract_mermaid_only(elements)
@@ -856,7 +1081,7 @@ def generate_output(elements: Union[Dict, List[Dict]],
 
     else:  #  MD, FORM, REPORT
         elements_md, elements_action = make_preamble(
-            obj_type=entity_type,
+            obj_type=target_type,
             search_string=search_string,
             output_format=output_format
         )
@@ -865,10 +1090,38 @@ def generate_output(elements: Union[Dict, List[Dict]],
             elements=elements,
             elements_action=elements_action,
             output_format=output_format,
-            entity_type=entity_type,
+            entity_type=target_type,
             extract_properties_func=extract_properties_func,
             get_additional_props_func=get_additional_props_func,
             columns_struct = columns_struct
         )
 
         return elements_md
+
+# Machine keys explanation
+# -------------------------
+# In pyegeria’s reporting system, each output column is defined with two identifiers:
+# - name: the human-readable display label (e.g., "GUID", "Display Name") shown in tables and reports.
+# - key: the stable, programmatic identifier (usually snake_case, e.g., guid, display_name) used internally.
+#
+# We refer to the key as the machine key. It is intended to be:
+# - Unique within a given row/spec: so downstream code can unambiguously reference values.
+# - Stable over time: it should not change with presentation tweaks or localization.
+# - Program-friendly: lowercase with underscores, matching the data model or derivation (e.g., header vs properties).
+#
+# Why DICT output uses machine keys
+# - Display names are for humans and may repeat across different sources of data in one row. For example,
+#   Collections may include a GUID from the element header (key: guid, display name: "GUID") and a related member GUID
+#   (key: GUID, display name: "GUID"). If the DICT used display names, one would overwrite the other.
+# - By using the machine key (column['key']) as the dictionary key, DICT output avoids these collisions and remains
+#   deterministic for programmatic consumers.
+#
+# Collision handling
+# - If, within an unusual spec, two columns share the same key, DICT generation adds a numeric suffix (e.g., guid_1)
+#   to preserve both values. This is a safeguard and such collisions should be avoided in specs.
+#
+# Guidance for spec authors
+# - Choose clear, stable machine keys in snake_case that map to actual data fields or well-defined derivations.
+# - Reserve display names for presentation; do not rely on them for programmatic access.
+# - When you need two visually identical labels (e.g., both "GUID"), ensure their keys differ (e.g., guid vs GUID) to
+#   reflect different sources/semantics.