pyegeria 5.4.0.dev6__py3-none-any.whl → 5.4.0.dev8__py3-none-any.whl

md_processing/md_commands/data_designer_commands.py

@@ -16,6 +16,7 @@ from md_processing.md_processing_utils.extraction_utils import (extract_command_
 from md_processing.md_processing_utils.md_processing_constants import (load_commands, ERROR)
 from pyegeria import DEBUG_LEVEL, body_slimmer
 from pyegeria.egeria_tech_client import EgeriaTech
+from pyegeria.output_formatter import (extract_mermaid_only, extract_basic_dict, generate_output)
 
 
 GERIA_METADATA_STORE = os.environ.get("EGERIA_METADATA_STORE", "active-metadata-store")
@@ -1422,7 +1423,7 @@ def process_data_collection_list_command(egeria_client: EgeriaTech, txt: str, di
     attributes = parsed_output['attributes']
 
     valid = parsed_output['valid']
-
+    print(Markdown(f"Performing {command}"))
     print(Markdown(parsed_output['display']))
 
     if directive == "display":
@@ -1485,7 +1486,7 @@ def process_data_structure_list_command(egeria_client: EgeriaTech, txt: str, dir
     attributes = parsed_output['attributes']
 
     valid = parsed_output['valid']
-
+    print(Markdown(f"Performing {command}"))
     print(Markdown(parsed_output['display']))
 
     if directive == "display":
@@ -1545,7 +1546,7 @@ def process_data_field_list_command(egeria_client: EgeriaTech, txt: str, directi
     attributes = parsed_output['attributes']
 
     valid = parsed_output['valid']
-
+    print(Markdown(f"Performing {command}"))
     print(Markdown(parsed_output['display']))
 
     if directive == "display":
@@ -1563,6 +1564,15 @@ def process_data_field_list_command(egeria_client: EgeriaTech, txt: str, directi
         search_string = attributes.get('Search String', {}).get('value', '*')
         output_format = attributes.get('Output Format', {}).get('value', 'LIST')
         detailed = attributes.get('Detailed', {}).get('value', False)
+        as_of_time = attributes.get('AsOfTime', {}).get('value', None)
+        effective_time = attributes.get('Effective Time', {}).get('value', None)
+        sort_order = attributes.get('Sort Order', {}).get('value', None)
+        order_property = attributes.get('Order Property', {}).get('value', None)
+        starts_with = attributes.get('Start With', {}).get('value', True)
+        ends_with = attributes.get('End With', {}).get('value', False)
+        ignore_case = attributes.get('Ignore Case', {}).get('value', False)
+        start_from = attributes.get('Start From', {}).get('value', 0)
+        page_size = attributes.get('Page Size', {}).get('value', None)
 
         try:
             if not valid:  # First validate the command before we process it
@@ -1571,7 +1581,19 @@ def process_data_field_list_command(egeria_client: EgeriaTech, txt: str, directi
                 return None
 
             list_md = f"\n# `{object_type}` with filter: `{search_string}`\n\n"
-            struct = egeria_client.find_data_fields(search_string, output_format=output_format)
+            body = {
+                "class": "FilterRequestBody",
+                "asOfTime": as_of_time,
+                "effectiveTime": effective_time,
+                "forLineage": False,
+                "forDuplicateProcessing" : False,
+                "limitResultsByStatus": ["ACTIVE"],
+                "sequencingOrder": sort_order,
+                "sequencingProperty": order_property,
+                "filter": search_string,
+                }
+            struct = egeria_client.find_data_fields_w_body(body, start_from, page_size, starts_with,
+                                                           ends_with, ignore_case,output_format)
 
             if output_format == "DICT":
                 list_md += f"```\n{json.dumps(struct, indent=4)}\n```\n"
@@ -1605,7 +1627,7 @@ def process_data_class_list_command(egeria_client: EgeriaTech, txt: str, directi
     attributes = parsed_output['attributes']
 
     valid = parsed_output['valid']
-
+    print(Markdown(f"Performing {command}"))
     print(Markdown(parsed_output['display']))
 
     if directive == "display":
@@ -1648,125 +1670,4 @@ def process_data_class_list_command(egeria_client: EgeriaTech, txt: str, directi
     else:
         return None
 
-# @logger.catch
-# def process_list_data_dictionary_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") ->
-# Optional[str]:
-#     """
-#     Processes a list data dictionary.
-#
-#     :param txt: A string representing the input cell to be processed for
-#         extracting term-related attributes.
-#     :param directive: an optional string indicating the directive to be used - display, validate or execute
-#     :return: A string summarizing the outcome of the processing.
-#     """
-#
-#
-#     if directive == "display":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return None
-#     elif directive == "validate":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return True
-#     elif directive == "process":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return egeria_client.get_term_by_guid(known_guid, output_format=output_format)
 
-#
-# def process_term_history_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") -> Optional[str]:
-#     """
-#     Processes a term history object_action by extracting key attributes such as
-#     term name and output format from the given text.
-#
-#     :param txt: A string representing the input cell to be processed for
-#         extracting term-related attributes.
-#     :param directive: an optional string indicating the directive to be used - display, validate or execute
-#     :return: A string summarizing the outcome of the processing.
-#     """
-#     from md_processing.md_processing_utils.common_md_utils import set_debug_level
-#
-#     object_action = extract_command(txt)
-#     set_debug_level(directive)
-#     print(Markdown(f"{pre_command} `{object_action}` with directive: `{directive}`"))
-#
-#     term_name = process_simple_attribute(txt, TERM_NAME_LABELS, ERROR)
-#     output_format = process_simple_attribute(txt, OUTPUT_LABELS)
-#     output_format = output_format.upper() if output_format else "MD"
-#     if output_format not in ELEMENT_OUTPUT_FORMATS:
-#         print_msg(WARNING, f"Output format {output_format} not recognized, using MD", debug_level)
-#         output_format = "MD"
-#
-#     if term_name is None:
-#         print_msg(ERROR, "No term name found", debug_level)
-#         return None
-#
-#     known_q_name, known_guid, valid, term_exists = process_element_identifiers(egeria_client, "Term",
-#                                                                               TERM_NAME_LABELS, txt,
-#                                                                               EXISTS_REQUIRED, None)
-#     if not term_exists:
-#         print_msg(ERROR, f"Term {term_name} not found", debug_level)
-#         return None
-#
-#     if directive == "display":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return None
-#     elif directive == "validate":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return True
-#     elif directive == "process":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return egeria_client.get_term_history(known_guid, output_format=output_format)
-#
-#
-# def process_term_revision_history_command(egeria_client: EgeriaTech, txt: str, directive: str = "display") ->
-# Optional[str]:
-#     """
-#     Processes a term revision history object_action by extracting key attributes such as
-#     term name and output format from the given text.
-#
-#     :param txt: A string representing the input cell to be processed for
-#         extracting term-related attributes.
-#     :param directive: an optional string indicating the directive to be used - display, validate or execute
-#     :return: A string summarizing the outcome of the processing.
-#     """
-#     from md_processing.md_processing_utils.common_md_utils import set_debug_level
-#
-#     object_action = extract_command(txt)
-#     set_debug_level(directive)
-#     print(Markdown(f"{pre_command} `{object_action}` with directive: `{directive}`"))
-#
-#     term_name = process_simple_attribute(txt, TERM_NAME_LABELS, ERROR)
-#     output_format = process_simple_attribute(txt, OUTPUT_LABELS)
-#     output_format = output_format.upper() if output_format else "MD"
-#     if output_format not in ELEMENT_OUTPUT_FORMATS:
-#         print_msg(WARNING, f"Output format {output_format} not recognized, using MD", debug_level)
-#         output_format = "MD"
-#
-#     if term_name is None:
-#         print_msg(ERROR, "No term name found", debug_level)
-#         return None
-#
-#     known_q_name, known_guid, valid, term_exists = process_element_identifiers(egeria_client, "Term",
-#                                                                               TERM_NAME_LABELS, txt,
-#                                                                               EXISTS_REQUIRED, None)
-#     if not term_exists:
-#         print_msg(ERROR, f"Term {term_name} not found", debug_level)
-#         return None
-#
-#     if directive == "display":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return None
-#     elif directive == "validate":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return True
-#     elif directive == "process":
-#         print(Markdown(f"\n* Command: {object_action}\n\t* Term Name: {term_name}\n\t* Output Format: {
-#         output_format}"))
-#         return egeria_client.get_term_revision_history(known_guid, output_format=output_format)

md_processing/md_processing_utils/generate_dr_help.py

@@ -0,0 +1,150 @@
+"""
+This file contains general utility functions for processing Egeria Markdown
+"""
+import json
+import os
+import sys
+from typing import List
+
+from loguru import logger
+from rich import print
+from rich.console import Console
+from rich.markdown import Markdown
+
+from commands.cat.dr_egeria_command_help import EGERIA_INBOX_PATH
+from md_processing.md_processing_utils.common_md_utils import (get_current_datetime_string, get_element_dictionary,
+                                                               update_element_dictionary,
+                                                               split_tb_string, str_to_bool, )
+from md_processing.md_processing_utils.extraction_utils import (process_simple_attribute, extract_attribute,
+                                                                get_element_by_name)
+from md_processing.md_processing_utils.md_processing_constants import (get_command_spec, load_commands, load_commands, COMMAND_DEFINITIONS)
+
+from md_processing.md_processing_utils.message_constants import (ERROR, INFO, WARNING, ALWAYS, EXISTS_REQUIRED)
+from pyegeria import EgeriaTech
+from pyegeria._globals import DEBUG_LEVEL
+from pyegeria.output_formatter import generate_entity_md_table
+
+# Constants
+EGERIA_WIDTH = int(os.environ.get("EGERIA_WIDTH", "200"))
+EGERIA_ROOT_PATH = os.environ.get("EGERIA_ROOT_PATH", ".")
+EGERIA_INBOX_PATH = os.environ.get("EGERIA_INBOX_PATH", ".")
+
+console = Console(width=EGERIA_WIDTH)
+
+debug_level = DEBUG_LEVEL
+global COMMAND_DEFINITIONS
+
+load_commands('commands.json')
+
+
+log_format = "D {time} | {level} | {function} | {line} | {message} | {extra}"
+logger.remove()
+logger.add(sys.stderr, level="INFO", format=log_format, colorize=True)
+full_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_INBOX_PATH, "data_designer_debug.log")
+# logger.add(full_file_path, rotation="1 day", retention="1 week", compression="zip", level="TRACE", format=log_format,
+#            colorize=True)
+logger.add("debug_log", rotation="1 day", retention="1 week", compression="zip", level="TRACE", format=log_format,
+           colorize=True)
+def yes_no(input: str)->str:
+    if type(input) is bool:
+        if input:
+            return "True"
+        else:
+            return "False"
+
+    input = input.title()
+    if input in ["Yes","No"]:
+        return input
+
+    if input in ["True", "T"]:
+        return "Yes"
+    else:
+        return "No"
+
+@logger.catch
+def _extract_help_fields(command: dict):
+    """
+
+    """
+
+    command_spec = get_command_spec(command)
+    attributes = command_spec.get('Attributes', [])
+    command_display_name = command_spec.get('display_name', None)
+    command_qn_prefix = command_spec.get('qn_prefix', None)
+
+    term_entry: list = []
+    for attr in attributes:
+        for key in attr:
+            if attr[key].get('level','Basic') != "Basic":
+                continue
+            attribute_name = key
+            input_required = yes_no(attr[key].get('input_required', "No"))
+
+            read_only = yes_no(attr[key].get('user_specified', "No"))
+            generated = yes_no(attr[key].get('generated', "No"))
+            default = attr[key].get('default', None)
+            if default:
+                generated = default
+            unique = yes_no(attr[key].get('unique', "No"))
+            notes = attr[key].get('description', "")
+            valid_values = attr[key].get('valid_values', [])
+            term_entry.append ({
+                'Attribute Name' : attribute_name,
+                'Input Required' : input_required,
+                'Read Only' : read_only,
+                'Generated' : generated,
+                'Default Value' : default,
+                'Notes' : notes,
+                'Unique Values' : unique,
+                'Valid Values' : valid_values,
+                })
+
+    return term_entry
+
+def create_help_terms():
+    term_entry:str = ""
+    glossary_name = "Egeria-Markdown"
+    commands = COMMAND_DEFINITIONS["Command Specifications"]
+    columns = [{'name': "Attribute Name", 'key': 'Attribute Name'},
+               {'name': 'Input Required', 'key': 'Input Required'},
+               {'name': 'Read Only', 'key': 'Read Only'},
+               {'name': 'Generated', 'key': 'Generated'},
+               {'name': 'Default Value', 'key': 'Default Value'},
+               {'name': 'Notes', 'key': 'Notes'},
+               {'name': 'Unique Values', 'key': 'Unique Values'},
+               {'name': 'Valid Values', 'key': 'Valid Values'}]
+
+    term_entry = """# Generating glossary entries for the documented commands\n\n
+            This file contains generated Dr.Egeria commands to generate glossary term entries describing
+            each command represented in the `commands.json` file.\n"""
+
+    for command, values in commands.items():
+        if command == "exported":
+            continue
+        if commands[command].get("level","") not in ["Basic"]:
+            continue
+        command_description = commands[command].get("description","")
+        command_verb = commands[command].get("verb","")
+
+        term_entry+= "# Create Term\n"
+        term_entry+= f"## Term Name\n\n{command_verb} {command}\n\n"
+        term_entry+= f"## Description\n\n{command_description}\n\n"
+        term_entry+= f"## Owning Glossary\n\n{glossary_name}\n\n"
+        term_entry+= f"## Categories\n\nWriting Dr.Egeria Markdown\n\n"
+
+
+        du = _extract_help_fields(command)
+        output = generate_entity_md_table(du, "", "", _extract_help_fields, columns, None, "help" )
+
+        term_entry+= f"## Usage\n\n{output}\n\n___\n\n"
+    print(term_entry)
+    file_path =  os.path.join(EGERIA_ROOT_PATH, EGERIA_INBOX_PATH, "generated_help_terms.md")
+    with open(file_path, 'a', encoding="utf-8") as f:
+        f.write(term_entry)
+
+def main():
+    create_help_terms()
+
+
+if __name__ == "__main__":
+    main()

md_processing/md_processing_utils/generate_md_templates.py

@@ -0,0 +1,92 @@
+"""
+This file contains general utility functions for processing Egeria Markdown
+"""
+import os
+import sys
+from typing import List
+
+from loguru import logger
+from rich import print
+from rich.console import Console
+
+from commands.cat.dr_egeria_command_help import EGERIA_INBOX_PATH
+from md_processing.md_processing_utils.common_md_utils import (get_current_datetime_string, get_element_dictionary,
+                                                               update_element_dictionary,
+                                                               split_tb_string, str_to_bool, )
+from md_processing.md_processing_utils.extraction_utils import (process_simple_attribute, extract_attribute,
+                                                                get_element_by_name)
+from md_processing.md_processing_utils.md_processing_constants import (get_command_spec, load_commands, load_commands, COMMAND_DEFINITIONS)
+
+from md_processing.md_processing_utils.message_constants import (ERROR, INFO, WARNING, ALWAYS, EXISTS_REQUIRED)
+from pyegeria import EgeriaTech
+from pyegeria._globals import DEBUG_LEVEL
+
+
+# Constants
+EGERIA_WIDTH = int(os.environ.get("EGERIA_WIDTH", "200"))
+EGERIA_ROOT_PATH = os.environ.get("EGERIA_ROOT_PATH", ".")
+EGERIA_INBOX_PATH = os.environ.get("EGERIA_INBOX_PATH", ".")
+
+console = Console(width=EGERIA_WIDTH)
+
+debug_level = DEBUG_LEVEL
+global COMMAND_DEFINITIONS
+
+load_commands('commands.json')
+
+
+log_format = "D {time} | {level} | {function} | {line} | {message} | {extra}"
+logger.remove()
+logger.add(sys.stderr, level="INFO", format=log_format, colorize=True)
+full_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_INBOX_PATH, "data_designer_debug.log")
+# logger.add(full_file_path, rotation="1 day", retention="1 week", compression="zip", level="TRACE", format=log_format,
+#            colorize=True)
+logger.add("debug_log", rotation="1 day", retention="1 week", compression="zip", level="TRACE", format=log_format,
+           colorize=True)
+@logger.catch
+def main():
+    """
+
+    """
+    glossary_name = "Egeria-Markdown"
+    commands = COMMAND_DEFINITIONS["Command Specifications"]
+    for command, values in commands.items():
+        if command == "exported":
+            continue
+        if commands[command].get("level","") not in ["Basic"]:
+            continue
+        command_description = commands[command].get("description","")
+        command_verb = commands[command].get("verb","")
+        print(f"\n# {command_verb} {command}\n>\t{command_description}")
+
+        attributes = commands[command]['Attributes']
+        for attribute in attributes:
+            for key, value in attribute.items():
+                if value.get('level',"") not in ["Basic"]:
+                    continue
+                user_specified =  value.get('user_specified', 'true') in ["true", "True"]
+
+                print(f"\n## {key}")
+
+                print(f">\tInput Required: {value.get('input_required', "false")}")
+
+                print(f">\tDescription: {value.get("description","")}")
+                labels = value.get("attr_labels", None)
+                if labels:
+                    print(f">\tAlternative Labels: {labels}")
+
+                valid_values = value.get("valid_values", None)
+                if valid_values:
+                    print(f">\tValid Values: {valid_values}")
+                default_value = value.get("default_value", None)
+                if default_value:
+                    print(f">\tDefault Value: {default_value}")
+        print("\n___\n")
+
+
+
+
+    #
+
+if __name__ == "__main__":
+    main()