pyegeria 5.3.5.1__tar.gz → 5.3.5.3__tar.gz

{pyegeria-5.3.5.1 → pyegeria-5.3.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: pyegeria
-Version: 5.3.5.1
+Version: 5.3.5.3
 Summary: A python client for Egeria
 License: Apache 2.0
 Keywords: egeria,metadata,governance
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: click
 Requires-Dist: httpx
 Requires-Dist: jupyter
+Requires-Dist: jupyter-notebook-parser (>=0.1.4,<0.2.0)
 Requires-Dist: mermaid-py
 Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0)
 Requires-Dist: pytest

{pyegeria-5.3.5.1 → pyegeria-5.3.5.3}/pyegeria/asset_catalog_omvs.py

@@ -544,6 +544,11 @@ class AssetCatalog(Client):
     async def _async_get_asset_lineage_graph(
         self,
         asset_guid: str,
+        effective_time: str = None,
+        as_of_time: str = None,
+        relationship_types: [str] = None,
+        limit_to_isc_q_name: str = None,
+        hilight_isc_q_name: str = None,
         start_from: int = 0,
         page_size: int = max_paging_size,
     ) -> str | dict:
@@ -552,6 +557,16 @@ class AssetCatalog(Client):
          ----------
          asset_guid : str
              The unique identity of the asset to get the graph for.
+         effective_time: str, default is None
+            Effective time to query on. If not specified, the current time is used.
+         as_of_time: str = None
+            as_of_time to query on. If not specified, the current time is used.
+         relationship_types: [str], default is None,
+            relationship types to include in the lineage graph. If not specified, all relationship types are included.
+         limit_to_isc_q_name: str = None,
+            if specified, filters results to only include information supply chains with the given name.
+         hilight_isc_q_name: str = None,
+            if specified, highlights the information supply chain with the given name.
 
          start_from : int, optional
              The index from which to start fetching the engine actions. Default is 0.
@@ -576,27 +591,48 @@ class AssetCatalog(Client):
             f"{self.platform_url}/servers/{self.view_server}/api/open-metadata/asset-catalog/assets/{asset_guid}/"
             f"as-lineage-graph?startFrom={start_from}&pageSize={page_size}"
         )
-
-        response = await self._async_make_request("POST", url)
+        body = {
+            "effectiveTime": effective_time,
+            "asOfTime": as_of_time,
+            "relationshipTypes": relationship_types,
+            "limitToISCQualifiedName": limit_to_isc_q_name,
+            "highlightISCQualifiedName": hilight_isc_q_name,
+            }
+        response = await self._async_make_request("POST", url, body_slimmer(body))
         return response.json().get("assetLineageGraph", NO_ASSETS_FOUND)
 
     def get_asset_lineage_graph(
         self,
         asset_guid: str,
+        effective_time: str = None,
+        as_of_time: str = None,
+        relationship_types: [str] = None,
+        limit_to_isc_q_name: str = None,
+        hilight_isc_q_name: str = None,
         start_from: int = 0,
         page_size: int = max_paging_size,
-    ) -> str | dict:
+        ) -> str | dict:
         """Return the asset lineage including a mermaid markdown string. Async Version.
          Parameters
          ----------
          asset_guid : str
              The unique identity of the asset to get the graph for.
+        effective_time: str, default is None
+            Effective time to query on. If not specified, the current time is used.
+        as_of_time: str = None
+            as_of_time to query on. If not specified, the current time is used.
+        relationship_types: [str], default is None,
+            relationship types to include in the lineage graph. If not specified, all relationship types are included.
+        limit_to_isc_q_name: str = None,
+            if specified, filters results to only include information supply chains with the given name.
+        hilight_isc_q_name: str = None,
+            if speficied, highlights the information supply chain with the given name.
 
          start_from : int, optional
              The index from which to start fetching the engine actions. Default is 0.
 
          page_size : int, optional
-             The maximum number of engine actions to fetch in a single request. Default is `max_paging_size`.
+             The maximum number of elements to fetch in a single request. Default is `max_paging_size`.
 
          Returns
          -------
@@ -613,29 +649,46 @@ class AssetCatalog(Client):
 
         loop = asyncio.get_event_loop()
         response = loop.run_until_complete(
-            self._async_get_asset_lineage_graph(asset_guid, start_from, page_size)
+            self._async_get_asset_lineage_graph(asset_guid, effective_time, as_of_time, relationship_types,
+                                                limit_to_isc_q_name, hilight_isc_q_name, start_from, page_size)
         )
         return response
 
     def get_asset_lineage_mermaid_graph(
         self,
         asset_guid: str,
+        effective_time: str = None,
+        as_of_time: str = None,
+        relationship_types: [str] = None,
+        limit_to_isc_q_name: str = None,
+        hilight_isc_q_name: str = None,
         start_from: int = 0,
         page_size: int = max_paging_size,
-    ) -> str:
-        """Return the lineage as mermaid markdown string. Async Version.
+        ) -> str:
+        """Return the asset lineage including a mermaid markdown string. Async Version.
          Parameters
          ----------
          asset_guid : str
              The unique identity of the asset to get the graph for.
+        effective_time: str, default is None
+            Effective time to query on. If not specified, the current time is used.
+        as_of_time: str = None
+            as_of_time to query on. If not specified, the current time is used.
+        relationship_types: [str], default is None,
+            relationship types to include in the lineage graph. If not specified, all relationship types are included.
+        limit_to_isc_q_name: str = None,
+            if specified, filters results to only include information supply chains with the given name.
+        hilight_isc_q_name: str = None,
+            if specified, highlights the information supply chain with the given name.
 
          start_from : int, optional
              The index from which to start fetching the engine actions. Default is 0.
 
          page_size : int, optional
-             The maximum number of engine actions to fetch in a single request. Default is `max_paging_size`.
+             The maximum number of elements to fetch in a single request.
+             Default is `max_paging_size`.
 
-         Returns
+        Returns
          -------
         str
              A mermaid string representing the lineage.
@@ -646,9 +699,12 @@ class AssetCatalog(Client):
          PropertyServerException
          UserNotAuthorizedException
 
-        """
+    """
 
-        asset_graph = self.get_asset_lineage_graph(asset_guid, start_from, page_size)
+        asset_graph = self.get_asset_lineage_graph(asset_guid, effective_time,
+                                                   as_of_time, relationship_types,
+                                                   limit_to_isc_q_name, hilight_isc_q_name,
+                                                   start_from, page_size)
         return asset_graph.get("mermaidGraph")
 
     async def _async_get_assets_by_metadata_collection_id(

pyegeria-5.3.5.3/pyegeria/commands/cat/freddie_jupyter.py

@@ -0,0 +1,124 @@
+"""
+This is an ongoing experiment in parsing and playing with Freddie docs
+"""
+import json
+
+from jupyter_notebook_parser import JupyterNotebookParser
+import nbformat
+import os
+import re
+from pyegeria import EgeriaTech
+from rich import box, print
+from rich.console import Console
+from rich.markdown import Markdown
+from rich.prompt import Prompt
+from rich.table import Table
+from freddie_utils import (extract_command, process_glossary_upsert_command, process_term_upsert_command,
+                        get_current_datetime_string)
+
+import click
+from pyegeria import EgeriaTech, body_slimmer, NO_GLOSSARIES_FOUND, NO_TERMS_FOUND, NO_ELEMENTS_FOUND
+from pyegeria._exceptions import (
+    InvalidParameterException,
+    PropertyServerException,
+    print_exception_response,
+)
+import datetime
+
+EGERIA_METADATA_STORE = os.environ.get("EGERIA_METADATA_STORE", "active-metadata-store")
+EGERIA_KAFKA_ENDPOINT = os.environ.get("KAFKA_ENDPOINT", "localhost:9092")
+EGERIA_PLATFORM_URL = os.environ.get("EGERIA_PLATFORM_URL", "https://localhost:9443")
+EGERIA_VIEW_SERVER = os.environ.get("EGERIA_VIEW_SERVER", "view-server")
+EGERIA_VIEW_SERVER_URL = os.environ.get(
+    "EGERIA_VIEW_SERVER_URL", "https://localhost:9443"
+)
+EGERIA_INTEGRATION_DAEMON = os.environ.get("EGERIA_INTEGRATION_DAEMON", "integration-daemon")
+EGERIA_INTEGRATION_DAEMON_URL = os.environ.get(
+    "EGERIA_INTEGRATION_DAEMON_URL", "https://localhost:9443"
+)
+EGERIA_ADMIN_USER = os.environ.get("ADMIN_USER", "garygeeke")
+EGERIA_ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "secret")
+EGERIA_USER = os.environ.get("EGERIA_USER", "erinoverview")
+EGERIA_USER_PASSWORD = os.environ.get("EGERIA_USER_PASSWORD", "secret")
+EGERIA_WIDTH = os.environ.get("EGERIA_WIDTH", 220)
+EGERIA_JUPYTER = os.environ.get("EGERIA_JUPYTER", False)
+EGERIA_HOME_GLOSSARY_GUID = os.environ.get("EGERIA_HOME_GLOSSARY_GUID", None)
+EGERIA_GLOSSARY_PATH = os.environ.get("EGERIA_GLOSSARY_PATH", None)
+EGERIA_ROOT_PATH = os.environ.get("EGERIA_ROOT_PATH", "/Users/dwolfson/localGit/egeria-v5-3/egeria-workspaces")
+EGERIA_FREDDIE_INBOX_PATH = os.environ.get("EGERIA_FREDDIE_INBOX_PATH", "exchange/loading-bay/freddies-inbox")
+EGERIA_FREDDIE_OUTBOX_PATH = os.environ.get("EGERIA_FREDDIE_OUTBOX_PATH", "exchange/distribution-hub/freddies-outbox")
+commands = ["Create Glossary", "Create Term", "Create Governance Definition", "Update Term"]
+console = Console(width=int(EGERIA_WIDTH))
+
+
+
+@click.command("process-jupyter")
+@click.option("--file-path", help="File path to notebook",
+              default="glossary_creation_experiment.ipynb")
+@click.option("--directive", default="display-only", help="How to process the file")
+@click.option("--server", default=EGERIA_VIEW_SERVER, help="Egeria view server to use.")
+@click.option(
+    "--url", default=EGERIA_VIEW_SERVER_URL, help="URL of Egeria platform to connect to"
+)
+@click.option("--userid", default=EGERIA_USER, help="Egeria user")
+@click.option("--user_pass", default=EGERIA_USER_PASSWORD, help="Egeria user password")
+def process_jupyter_notebook(
+        file_path: str,
+        directive: str,
+        server: str,
+        url: str,
+        userid: str,
+        user_pass: str,
+        ):
+    client = EgeriaTech(server, url, user_id=userid)
+    token = client.create_egeria_bearer_token(userid, user_pass)
+    #
+    # parsed_notebook = JupyterNotebookParser(file_path)
+    #
+    # cell_sources = parsed_notebook.get_markdown_cell_sources()
+    try:
+        updated = False
+        full_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_FREDDIE_INBOX_PATH, file_path)
+        click.echo(f"Processing notebook: {full_file_path}")
+        with open(full_file_path, 'r') as f:
+            nb = nbformat.read(f, as_version=4)
+            for cell in nb.cells:
+                if cell.cell_type == 'markdown':
+                    if cell.source.strip().startswith('#'):
+                        potential_command = extract_command(cell.source)
+                        if potential_command in commands:
+                            if potential_command in ["Create Glossary", "Update Glossary"]:
+                                result = process_glossary_upsert_command(client, cell.source, directive)
+                                if result:
+                                    if directive == "process":
+                                        updated = True
+                                        cell.source = result
+                                else:
+                                    continue
+                            if potential_command in ["Create Term", "Update Term"]:
+                                result = process_term_upsert_command(client, cell.source, directive)
+                                if result:
+                                    if directive == "process":
+                                        updated = True
+                                        cell.source = result
+                                else:
+                                    continue
+
+        if updated:
+            path, filename = os.path.split(file_path)  # Get both parts
+            new_filename = f"processed-{get_current_datetime_string()}-{filename}"  # Create the new filename
+            new_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_FREDDIE_OUTBOX_PATH, new_filename)  # Construct the new path
+            os.makedirs(os.path.dirname(new_file_path), exist_ok=True)
+
+
+            with open(new_file_path, 'w') as f2:
+                nbformat.write(nb, f2)
+            click.echo(f"\n==>Notebook written to {new_file_path}")
+
+    except Exception as e:
+        print_exception_response(e)
+        return
+
+
+if __name__ == "__main__":
+    process_jupyter_notebook()

pyegeria-5.3.5.3/pyegeria/commands/cat/freddie_md.py

@@ -0,0 +1,158 @@
+"""
+This is an ongoing experiment in parsing and playing with Freddie docs
+"""
+import json
+import mistune
+from jupyter_notebook_parser import JupyterNotebookParser
+import nbformat
+import os
+import re
+from pyegeria import EgeriaTech
+from rich import box, print
+from rich.console import Console
+from rich.markdown import Markdown
+from rich.prompt import Prompt
+from rich.table import Table
+from freddie_utils import (extract_command, process_glossary_upsert_command, process_term_upsert_command,
+                        get_current_datetime_string, process_per_proj_upsert_command, commands)
+
+import click
+from pyegeria import EgeriaTech, body_slimmer, NO_GLOSSARIES_FOUND, NO_TERMS_FOUND, NO_ELEMENTS_FOUND
+from pyegeria._exceptions import (
+    InvalidParameterException,
+    PropertyServerException,
+    print_exception_response,
+)
+import datetime
+
+EGERIA_METADATA_STORE = os.environ.get("EGERIA_METADATA_STORE", "active-metadata-store")
+EGERIA_KAFKA_ENDPOINT = os.environ.get("KAFKA_ENDPOINT", "localhost:9092")
+EGERIA_PLATFORM_URL = os.environ.get("EGERIA_PLATFORM_URL", "https://localhost:9443")
+EGERIA_VIEW_SERVER = os.environ.get("EGERIA_VIEW_SERVER", "view-server")
+EGERIA_VIEW_SERVER_URL = os.environ.get(
+    "EGERIA_VIEW_SERVER_URL", "https://localhost:9443"
+)
+EGERIA_INTEGRATION_DAEMON = os.environ.get("EGERIA_INTEGRATION_DAEMON", "integration-daemon")
+EGERIA_INTEGRATION_DAEMON_URL = os.environ.get(
+    "EGERIA_INTEGRATION_DAEMON_URL", "https://localhost:9443"
+)
+EGERIA_ADMIN_USER = os.environ.get("ADMIN_USER", "garygeeke")
+EGERIA_ADMIN_PASSWORD = os.environ.get("ADMIN_PASSWORD", "secret")
+EGERIA_USER = os.environ.get("EGERIA_USER", "erinoverview")
+EGERIA_USER_PASSWORD = os.environ.get("EGERIA_USER_PASSWORD", "secret")
+EGERIA_WIDTH = os.environ.get("EGERIA_WIDTH", 220)
+EGERIA_JUPYTER = os.environ.get("EGERIA_JUPYTER", False)
+EGERIA_HOME_GLOSSARY_GUID = os.environ.get("EGERIA_HOME_GLOSSARY_GUID", None)
+EGERIA_GLOSSARY_PATH = os.environ.get("EGERIA_GLOSSARY_PATH", None)
+EGERIA_ROOT_PATH = os.environ.get("EGERIA_ROOT_PATH", "/Users/dwolfson/localGit/egeria-v5-3/egeria-workspaces")
+EGERIA_FREDDIE_INBOX_PATH = os.environ.get("EGERIA_FREDDIE_INBOX_PATH", "exchange/loading-bay/freddies-inbox")
+EGERIA_FREDDIE_OUTBOX_PATH = os.environ.get("EGERIA_FREDDIE_OUTBOX_PATH", "exchange/distribution-hub/freddies-outbox")
+console = Console(width=int(EGERIA_WIDTH))
+
+
+
+@click.command("process-markdown-file", help="Process a markdown file and return the output as a string.")
+@click.option("--file-path", help="File path to markdown file",
+              default="glossary_exp.md")
+@click.option("--directive", default="display", help="How to process the file")
+@click.option("--server", default=EGERIA_VIEW_SERVER, help="Egeria view server to use.")
+@click.option(
+    "--url", default=EGERIA_VIEW_SERVER_URL, help="URL of Egeria platform to connect to"
+)
+@click.option("--userid", default=EGERIA_USER, help="Egeria user")
+@click.option("--user_pass", default=EGERIA_USER_PASSWORD, help="Egeria user password")
+def process_markdown_file(
+        file_path: str,
+        directive: str,
+        server: str,
+        url: str,
+        userid: str,
+        user_pass: str,
+        ):
+    client = EgeriaTech(server, url, user_id=userid)
+    token = client.create_egeria_bearer_token(userid, user_pass)
+
+    updated = False
+    full_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_FREDDIE_INBOX_PATH, file_path)
+    print(f"Processing Markdown File: {full_file_path}")
+    try:
+        with open(full_file_path, 'r') as f:
+            lines = f.readlines()
+    except FileNotFoundError:
+        print(f"Error: File not found at path: {full_file_path}")
+        return {}  # Return empty dict if file not found
+
+    h1_blocks = []
+    current_block = ""
+    in_h1_block = False
+
+    for line in lines:
+        line = line.strip()  # Remove leading/trailing whitespace
+
+        if line.startswith("# ") and not in_h1_block:  # Start of a new H1 block
+            if current_block:
+                h1_blocks.append(current_block)
+            current_block = line
+            in_h1_block = True
+        elif line == "---" and in_h1_block:  # End of the current H1 block
+            h1_blocks.append(current_block)
+            current_block = ""
+            in_h1_block = False
+        elif in_h1_block:  # Add line to the current H1 block
+            current_block += "\n" + line
+
+    if current_block:  # Add the last H1 block
+        h1_blocks.append(current_block)
+
+    processed_blocks = ''
+    for block in h1_blocks:
+        potential_command = extract_command(block)
+
+        if potential_command in commands:
+            if potential_command in ["Create Glossary", "Update Glossary"]:
+                result = process_glossary_upsert_command(client, block, directive)
+                if result:
+                     if directive == "process":
+                        updated = True
+                        processed_blocks += f'\n---\n{result} \n---\n'
+                else:
+                    print(f"\n==>\tErrors found while processing command: \'{potential_command}\'\n"
+                          f"\tPlease correct and try again. \n")
+                    continue
+            if potential_command in ["Create Term", "Update Term"]:
+                result = process_term_upsert_command(client, block, directive)
+                if result:
+                    if directive == "process":
+                        updated = True
+                        processed_blocks += f'\n---\n{result} \n---\n\n'
+                else:
+                    print(f"\n==>\tErrors found while processing command: \'{potential_command}\'\n"
+                          f"\tPlease correct and try again. \n")
+                    continue
+
+            if potential_command in ["Create Personal Project", "Update Personal Project"]:
+                result = process_per_proj_upsert_command(client, block, directive)
+                if result:
+                    if directive == "process":
+                        updated = True
+                        processed_blocks += f'\n---\n{result} \n---\n\n'
+                else:
+                    print(f"\n==>\tErrors found while processing command: \'{potential_command}\'\n"
+                          f"\tPlease correct and try again. \n")
+                    continue
+
+    if updated:
+        path, filename = os.path.split(file_path)  # Get both parts
+        new_filename = f"processed-{get_current_datetime_string()}-{filename}"  # Create the new filename
+        new_file_path = os.path.join(EGERIA_ROOT_PATH, EGERIA_FREDDIE_OUTBOX_PATH, new_filename)  # Construct the new path
+        os.makedirs(os.path.dirname(new_file_path), exist_ok=True)
+
+
+        with open(new_file_path, 'w') as f2:
+            f2.write(processed_blocks)
+        click.echo(f"\n==>Notebook written to {new_file_path}")
+
+
+
+if __name__ == "__main__":
+    process_markdown_file()