monocle-apptrace 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

monocle_apptrace/{wrapper_config/lang_chain_methods.json → metamodel/maps/langchain_methods.json}

@@ -1,4 +1,4 @@
-{ 
+{
 "wrapper_methods" : [
     {
         "package": "langchain.prompts.base",
@@ -19,42 +19,65 @@
         "object": "BaseChatModel",
         "method": "invoke",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "llm_wrapper"
+        "wrapper_method": "llm_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/langchain_entities.json"]
     },
     {
         "package": "langchain.chat_models.base",
         "object": "BaseChatModel",
         "method": "ainvoke",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "allm_wrapper"
+        "wrapper_method": "allm_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/langchain_entities.json"]
     },
     {
         "package": "langchain_core.language_models.llms",
         "object": "LLM",
         "method": "_generate",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "llm_wrapper"
+        "wrapper_method": "llm_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/langchain_entities.json"]
     },
     {
         "package": "langchain_core.language_models.llms",
         "object": "LLM",
         "method": "_agenerate",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "llm_wrapper"
+        "wrapper_method": "allm_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/langchain_entities.json"]
+    },
+    {
+        "package": "langchain_core.language_models.llms",
+        "object": "BaseLLM",
+        "method": "invoke",
+        "wrapper_package": "wrap_common",
+        "wrapper_method": "llm_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/langchain_entities.json"]
+    },
+    {
+        "package": "langchain_core.language_models.llms",
+        "object": "BaseLLM",
+        "method": "ainvoke",
+        "wrapper_package": "wrap_common",
+        "wrapper_method": "allm_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/langchain_entities.json"]
     },
     {
         "package": "langchain_core.retrievers",
         "object": "BaseRetriever",
         "method": "invoke",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "task_wrapper"
+        "wrapper_method": "task_wrapper",
+        "output_processor": ["metamodel/maps/attributes/retrieval/langchain_entities.json"]
+
     },
     {
         "package": "langchain_core.retrievers",
         "object": "BaseRetriever",
         "method": "ainvoke",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "atask_wrapper"
+        "wrapper_method": "atask_wrapper",
+        "output_processor": ["metamodel/maps/attributes/retrieval/langchain_entities.json"]
     },
     {
         "package": "langchain.schema",
@@ -103,4 +126,4 @@
         "wrapper_method": "atask_wrapper"
     }
 ]
-}
+}

monocle_apptrace/{wrapper_config/llama_index_methods.json → metamodel/maps/llamaindex_methods.json}

@@ -6,7 +6,8 @@
         "method": "retrieve",
         "span_name": "llamaindex.retrieve",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "task_wrapper"
+        "wrapper_method": "task_wrapper",
+        "output_processor": ["metamodel/maps/attributes/retrieval/llamaindex_entities.json"]
     },
     {
         "package": "llama_index.core.indices.base_retriever",
@@ -14,7 +15,8 @@
         "method": "aretrieve",
         "span_name": "llamaindex.retrieve",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "atask_wrapper"
+        "wrapper_method": "atask_wrapper",
+        "output_processor": ["metamodel/maps/attributes/retrieval/llamaindex_entities.json"]
     },
     {
         "package": "llama_index.core.base.base_query_engine",
@@ -38,7 +40,8 @@
         "method": "chat",
         "span_name": "llamaindex.llmchat",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "task_wrapper"
+        "wrapper_method": "task_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/llamaindex_entities.json"]
     },
     {
         "package": "llama_index.core.llms.custom",
@@ -46,7 +49,8 @@
         "method": "achat",
         "span_name": "llamaindex.llmchat",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "atask_wrapper"
+        "wrapper_method": "atask_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/llamaindex_entities.json"]
     },
     {
         "package": "llama_index.llms.openai.base",
@@ -55,8 +59,7 @@
         "span_name": "llamaindex.openai",
         "wrapper_package": "wrap_common",
         "wrapper_method": "llm_wrapper",
-        "span_name_getter_package" : "llamaindex",
-        "span_name_getter_mothod" : "get_llm_span_name_for_openai"
+        "output_processor": ["metamodel/maps/attributes/inference/llamaindex_entities.json"]
     },
     {
         "package": "llama_index.llms.openai.base",
@@ -64,7 +67,8 @@
         "method": "achat",
         "span_name": "llamaindex.openai",
         "wrapper_package": "wrap_common",
-        "wrapper_method": "allm_wrapper"
+        "wrapper_method": "allm_wrapper",
+        "output_processor": ["metamodel/maps/attributes/inference/llamaindex_entities.json"]
     }
 ]
-}
+}

monocle_apptrace/metamodel/spans/README.md

@@ -0,0 +1,121 @@
+# Monocle Span format
+Monocle generates [traces](../../../../Monocle_User_Guide.md#traces) which comprises of [spans](../../../../Monocle_User_Guide.md#spans). Note that Monocle trace is [OpenTelemetry format](https://opentelemetry.io/docs/concepts/signals/traces/) compatible. Each span is essentially a step in the execution that interacts with one of more GenAI technology components. This document explains the [span format](./span_format.json) that Monocle generates for GenAI application tracing.
+
+Per the OpenTelemetry convention, each span contains an attribute section and event section. In Monocle generated trace, the attribute sections includes details of GenAI entities used in the span. The event section includes the input, output and metadata related to the execution of that span.
+
+## Attributes 
+The attribute sections includes details of GenAI entities used in the span. For each entity used in the span in includes the entity name and entity type. For every type of entity, there are required and optional attributes listed below.
+### Json format
+```json
+    attributes: 
+        "span.type": "Monocle-span-type",
+        "entity.count": "count-of-entities",
+    
+        "entity.<index>.name": "Monocle-Entity-name",
+        "entity.<index>.type": "MonocleEntity.<entity-type>"
+        ...
+```
+The ```entity.count``` indicates total number of entities used in the given span. For each entity, the details are captured in ```entity.<index>.X```. For example, 
+```json
+    "attributes": {
+        "span.type": "Inference",
+        "entity.count": 2,
+        "entity.1.name": "AzureOpenAI",
+        "entity.1.type": "Inference.Azure_oai",
+        "entity.2.name": "gpt-35-turbo",
+        "entity.2.type": "Model.LLM",
+        "entity.2.model_name": "gpt-35-turbo",
+```
+
+### Entity type specific attributes
+#### MonocleEntity.Workflow
+| Name | Description | Values | Required |
+| - | - | - | - |
+| name | Entity name generated by Monocle | Name String | Required |
+| type | Monocle Entity type | MonocleEntity.Workflow | Required |
+| optional-attribute | Additional attribute specific to entity |  | Optional |
+
+### MonocleEntity.Model
+| Name | Description | Values | Required |
+| - | - | - | - |
+| name | Entity name generated by Monocle | Name String | Required |
+| type | Monocle Entity type | MonocleEntity.Model | Required |
+| model_name | Name of model | String | Required |
+| optional-attribute | Additional attribute specific to entity |  | Optional |
+
+### MonocleEntity.AppHosting
+| Name | Description | Values | Required |
+| - | - | - | - |
+| name | Entity name generated by Monocle | Name String | Required |
+| type | Monocle Entity type | MonocleEntity.AppHosting | Required |
+| optional-attribute | Additional attribute specific to entity |  | Optional |
+
+### MonocleEntity.Inference
+| Name | Description | Values | Required |
+| - | - | - | - |
+| name | Entity name generated by Monocle | Name String | Required |
+| type | Monocle Entity type | MonocleEntity.Inference | Required |
+| optional-attribute | Additional attribute specific to entity |  | Optional |
+
+### MonocleEntity.VectorDB
+| Name | Description | Values | Required |
+| - | - | - | - |
+| name | Entity name generated by Monocle | Name String | Required |
+| type | Monocle Entity type | MonocleEntity.VectorDB | Required |
+| optional-attribute | Additional attribute specific to entity |  | Optional |
+
+## Events
+The event section includes the input, output and metadata generated by that span execution. For each type of span, there are required and option input, output and metadata items listed below. If there's no data genearated in the space, the events will be an empty array.
+
+### Json format
+```json
+    "events" : [
+        {
+            "name": "data.input",
+            "timestamp": "UTC timestamp",
+            "attributes": {
+                "input_attribute": "value"
+           }
+        },
+        {
+            "name": "data.output",
+            "timestamp": "UTC timestamp",
+            "attributes": {
+                "output_attribute": "value"
+            }
+        },
+        {
+            "name": "metadata",
+            "timestamp": "UTC timestamp",
+            "attributes": {
+                "metadata_attribute": "value"
+            }
+        }
+    ]
+```
+
+## Span types and events
+The ```span.type``` captured in ```attributes``` section of the span dectates the format of the ```events```
+### SpanType.Retrieval
+| Name | Description | Values | Required |
+| - | - | - | - |
+| name | event name  | data.input or data.output or metadata | Required |
+| timestamp | timestap when the event occurred | UTC timestamp | Required |
+| attributes | input/output/metadata attributes generated in span | Dictionary | Required |
+
+### SpanType.Inference
+| Name | Description | Values | Required |
+| - | - | - | - |
+| name | event name  | data.input or data.output or metadata | Required |
+| timestamp | timestap when the event occurred | UTC timestamp | Required |
+| attributes | input/output/metadata attributes generated in span | Dictionary | Required |
+
+### SpanType.Workflow
+| Name | Description | Values | Required |
+| - | - | - | - |
+| name | event name  | data.input or data.output or metadata | Required |
+| timestamp | timestap when the event occurred | UTC timestamp | Required |
+| attributes | input/output/metadata attributes generated in span | Dictionary | Required |
+
+### SpanType.Internal
+Events will be empty

monocle_apptrace/metamodel/spans/span_example.json

@@ -0,0 +1,140 @@
+{
+    "name": "llamaindex.retrieve",
+    "context": {
+        "trace_id": "0x93cd0bf865b3ffcc3cf9c075dc3e3797",
+        "span_id": "0x5d3f839e900bda24",
+        "trace_state": "[]"
+    },
+    "kind": "SpanKind.CLIENT",
+    "parent_id": "0x7a63d63e42ccac60",
+    "start_time": "2024-09-09T14:38:45.237182Z",
+    "end_time": "2024-09-09T14:38:45.620112Z",
+    "status": {
+        "status_code": "OK"
+    },
+    "attributes": {
+        "span.type": "retrieval", 
+        "entity.count": 2,
+        "entity.1.name": "ChromaVectorStore",
+        "entity.1.type": "vectorstore.chroma",
+        "entity.1.embedding-model-name": "BAAI/bge-small-en-v1.5",
+        "entity.2.name": "BAAI/bge-small-en-v1.5",
+        "entity.2.type": "model.embedding",
+        "entity.2.model_name": "BAAI/bge-small-en-v1.5"
+    },
+    "events": [
+        {
+            "name": "data.input",
+            "timestamp": "timestamp",
+            "attributes": {
+                "context_input": "question: What is an americano?"
+            }
+        },
+        {
+            "name": "data.output",
+            "timestamp": "timestamp",
+            "attributes": {
+                "context_output": "Coffee is a hot drink made from the roasted and ground seeds (coffee beans) of a tropical shrub\nA latte consists of one or more shots of espresso, served in a glass (or sometimes a cup), into which hot steamed milk is added\nAmericano is a type of coffee drink prepared by diluting an espresso shot with hot water at a 1:3 to 1:4 ratio, resulting in a drink that retains the complex flavors of espresso, but in a lighter way"
+            }
+        }
+
+    ],
+    "links": [],
+    "resource": {
+        "attributes": {
+            "service.name": "coffee-bot"
+        },
+        "schema_url": ""
+    }
+},
+{
+    "name": "llamaindex.openai",
+    "context": {
+        "trace_id": "0x93cd0bf865b3ffcc3cf9c075dc3e3797",
+        "span_id": "0x8b6363e1937a4d7b",
+        "trace_state": "[]"
+    },
+    "kind": "SpanKind.CLIENT",
+    "parent_id": "0x7a63d63e42ccac60",
+    "start_time": "2024-09-09T14:38:45.622174Z",
+    "end_time": "2024-09-09T14:38:46.514120Z",
+    "status": {
+        "status_code": "OK"
+    },
+    "attributes": {
+        "span.type": "inference",
+        "entity.count": 2,
+        "entity.1.name": "AzureOpenAI",
+        "entity.1.type": "inference.azure_oai",
+        "entity.1.provider_name": "openai.azure.com",
+        "entity.1.deployment": "kshitiz-gpt",
+        "entity.1.inference_endpoint": "https://okahu-openai-dev.openai.azure.com/",
+
+        "entity.2.name": "gpt-35-turbo",
+        "entity.2.type": "model.llm",
+        "entity.2.model_name": "gpt-35-turbo"
+    },
+    "events": [
+        {
+            "name": "data.input",
+            "timestamp": "timestamp",
+            "attributes": {
+                "question": "What is an americano?",
+            }
+        },
+        {
+            "name": "data.output",
+            "timestamp": "timestamp",
+            "attributes": {
+                "response": "An americano is a type of coffee drink that is made by diluting an espresso shot with hot water at a 1:3 to 1:4 ratio, resulting in a drink that retains the complex flavors of espresso, but in a lighter way.",
+            }
+        },
+        {
+            "name": "metadata",
+            "timestamp": "timestamp",
+            "attributes": {
+                "temperature": 0.1,
+                "completion_tokens": 52,
+                "prompt_tokens": 233,
+                "total_tokens": 285
+            }
+        }
+    ],
+    "links": [],
+    "resource": {
+        "attributes": {
+            "service.name": "coffee-bot"
+        },
+        "schema_url": ""
+    }
+}
+{
+    "name": "llamaindex.query",
+    "context": {
+        "trace_id": "0x93cd0bf865b3ffcc3cf9c075dc3e3797",
+        "span_id": "0x7a63d63e42ccac60",
+        "trace_state": "[]"
+    },
+    "kind": "SpanKind.CLIENT",
+    "parent_id": null,
+    "start_time": "2024-09-09T14:38:45.236627Z",
+    "end_time": "2024-09-09T14:38:46.514442Z",
+    "status": {
+        "status_code": "OK"
+    },
+    "attributes": {
+        "span.type": "workflow",
+        "entity.count": 1,
+        "entity.1.name": "coffee-bot",
+        "entity.1.type": "workflow.llama_index"
+    },
+    "events": [
+    ],
+    "links": [],
+    "resource": {
+        "attributes": {
+            "service.name": "coffee-bot"
+        },
+        "schema_url": ""
+    }
+}

monocle_apptrace/metamodel/spans/span_format.json

@@ -0,0 +1,55 @@
+{
+    "name": "span-name",
+    "context": {
+        "trace_id": "trace-id",
+        "span_id": "span-id",
+        "trace_state": "[]"
+    },
+    "kind": "SpanKind.CLIENT",
+    "parent_id": "parent-id or None (for root span)",
+    "start_time": "UTC timestamp",
+    "end_time": "UTC timestamp",
+    "status": {
+        "status_code": "OK or Error"
+    },
+    "attributes": {
+        "description": "List of AI component entities used in this span, eg Model, Inference hosting service. Needs to be one of the supported entity types.",
+    
+        "span.type": "Monocle-span-type",
+        "entity.count": "count-of-entities",
+    
+        "entity.<index>.name": "Monocle-Entity-name",
+        "entity.<index>.type": "Monocle-Entity-Type",
+        "entity.<index>.<attribute>": "Value"
+    },
+    "events" : [
+        {
+            "name": "data.input",
+            "timestamp": "UTC timestamp",
+            "attributes": {
+                "input_attribute": "value"
+           }
+        },
+        {
+            "name": "data.output",
+            "timestamp": "UTC timestamp",
+            "attributes": {
+                "output_attribute": "value"
+            }
+        },
+        {
+            "name": "metadata",
+            "timestamp": "UTC timestamp",
+            "attributes": {
+                "metadata_attribute": "value"
+            }
+        }
+    ],
+    "links": [],
+    "resource": {
+        "attributes": {
+            "service.name": "top-workflow-name"
+        },
+        "schema_url": ""
+    }
+}

monocle_apptrace/metamodel/spans/span_types.json

@@ -0,0 +1,16 @@
+{
+    "span_types" : [
+        {
+            "type": "inference",
+            "description": "Model inference span"
+        },
+        {
+            "type": "retrieval",
+            "description": "vector embedding retrieval"
+        },
+        {
+            "type": "workflow",
+            "description": "workflow orchetraction at top level"
+        }
+    ]
+}

monocle_apptrace/utils.py

@@ -3,7 +3,12 @@ import json
 from importlib import import_module
 import os
 from opentelemetry.trace import Span
+from opentelemetry.context import attach, set_value, get_value
 from monocle_apptrace.constants import azure_service_map, aws_service_map
+from json.decoder import JSONDecodeError
+logger = logging.getLogger(__name__)
+
+embedding_model_context = {}
 
 def set_span_attribute(span, name, value):
     if value is not None:
@@ -19,12 +24,14 @@ def dont_throw(func):
     """
     # Obtain a logger specific to the function's module
     logger = logging.getLogger(func.__module__)
+
     # pylint: disable=inconsistent-return-statements
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as ex:
             logger.warning("Failed to execute %s, error: %s", func.__name__, str(ex))
+
     return wrapper
 
 def with_tracer_wrapper(func):
@@ -46,28 +53,120 @@ def resolve_from_alias(my_map, alias):
             return my_map[i]
     return None
 
-def load_wrapper_from_config(config_file_path: str, module_name: str = None):
-    wrapper_methods = []
-    with open(config_file_path, encoding='UTF-8') as config_file:
+def load_output_processor(wrapper_method, attributes_config_base_path):
+    """Load the output processor from a file if the file path is provided and valid."""
+    logger = logging.getLogger()
+    output_processor_file_path = wrapper_method["output_processor"][0]
+    logger.info(f'Output processor file path is: {output_processor_file_path}')
+
+    if isinstance(output_processor_file_path, str) and output_processor_file_path:  # Combined condition
+        if not attributes_config_base_path:
+            absolute_file_path = os.path.abspath(output_processor_file_path)
+        else:
+            absolute_file_path = os.path.join(attributes_config_base_path, output_processor_file_path)
+
+        logger.info(f'Absolute file path is: {absolute_file_path}')
+        try:
+            with open(absolute_file_path, encoding='UTF-8') as op_file:
+                wrapper_method["output_processor"] = json.load(op_file)
+                logger.info('Output processor loaded successfully.')
+        except FileNotFoundError:
+            logger.error(f"Error: File not found at {absolute_file_path}.")
+        except JSONDecodeError:
+            logger.error(f"Error: Invalid JSON content in the file {absolute_file_path}.")
+        except Exception as e:
+            logger.error(f"Error: An unexpected error occurred: {e}")
+    else:
+        logger.error("Invalid or missing output processor file path.")
+
+def get_wrapper_methods_config(
+        wrapper_methods_config_path: str,
+        attributes_config_base_path: str = None
+):
+    parent_dir = os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
+    wrapper_methods_config = load_wrapper_methods_config_from_file(
+        wrapper_methods_config_path=os.path.join(parent_dir, wrapper_methods_config_path))
+    process_wrapper_method_config(
+        wrapper_methods_config=wrapper_methods_config,
+        attributes_config_base_path=attributes_config_base_path)
+    return wrapper_methods_config
+
+def load_wrapper_methods_config_from_file(
+        wrapper_methods_config_path: str):
+    json_data = {}
+
+    with open(wrapper_methods_config_path, encoding='UTF-8') as config_file:
         json_data = json.load(config_file)
-        wrapper_methods = json_data["wrapper_methods"]
-        for wrapper_method in wrapper_methods:
+
+    return json_data["wrapper_methods"]
+
+def process_wrapper_method_config(
+        wrapper_methods_config: str,
+        attributes_config_base_path: str = ""):
+    for wrapper_method in wrapper_methods_config:
+        if "wrapper_package" in wrapper_method and "wrapper_method" in wrapper_method:
             wrapper_method["wrapper"] = get_wrapper_method(
                 wrapper_method["wrapper_package"], wrapper_method["wrapper_method"])
-            if "span_name_getter_method" in wrapper_method :
+            if "span_name_getter_method" in wrapper_method:
                 wrapper_method["span_name_getter"] = get_wrapper_method(
                     wrapper_method["span_name_getter_package"],
                     wrapper_method["span_name_getter_method"])
-        return wrapper_methods
+        if "output_processor" in wrapper_method and wrapper_method["output_processor"]:
+            load_output_processor(wrapper_method, attributes_config_base_path)
 
 def get_wrapper_method(package_name: str, method_name: str):
     wrapper_module = import_module("monocle_apptrace." + package_name)
     return getattr(wrapper_module, method_name)
 
 def update_span_with_infra_name(span: Span, span_key: str):
-    for key,val  in azure_service_map.items():
+    for key, val in azure_service_map.items():
         if key in os.environ:
             span.set_attribute(span_key, val)
-    for key,val  in aws_service_map.items():
+    for key, val in aws_service_map.items():
         if key in os.environ:
             span.set_attribute(span_key, val)
+
+def set_embedding_model(model_name: str):
+    """
+    Sets the embedding model in the global context.
+
+    @param model_name: The name of the embedding model to set
+    """
+    embedding_model_context['embedding_model'] = model_name
+
+def get_embedding_model() -> str:
+    """
+    Retrieves the embedding model from the global context.
+
+    @return: The name of the embedding model, or 'unknown' if not set
+    """
+    return embedding_model_context.get('embedding_model', 'unknown')
+
+def set_attribute(key: str, value: str):
+    """
+    Set a value in the global context for a given key.
+
+    Args:
+        key: The key for the context value to set.
+        value: The value to set for the given key.
+    """
+    attach(set_value(key, value))
+
+def get_attribute(key: str) -> str:
+    """
+    Retrieve a value from the global context for a given key.
+
+    Args:
+        key: The key for the context value to retrieve.
+
+    Returns:
+        The value associated with the given key.
+    """
+    return get_value(key)
+
+def get_workflow_name(span: Span) -> str:
+    try:
+        return get_value("workflow_name") or span.resource.attributes.get("service.name")
+    except Exception as e:
+        logger.exception(f"Error getting workflow name: {e}")
+        return None