monocle-apptrace 0.1.1__py3-none-any.whl → 0.3.0b1__py3-none-any.whl

monocle_apptrace/langchain/__init__.py

@@ -1,6 +1,9 @@
 import os
-from monocle_apptrace.utils import load_wrapper_from_config
+from monocle_apptrace.utils import get_wrapper_methods_config
 
 parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-LANGCHAIN_METHODS = load_wrapper_from_config(
-    os.path.join(parent_dir, 'metamodel', 'maps', 'lang_chain_methods.json'))
+LANGCHAIN_METHODS = get_wrapper_methods_config(
+    wrapper_methods_config_path=os.path.join(parent_dir, 'metamodel', 'maps', 'langchain_methods.json'),
+    attributes_config_base_path=os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+

monocle_apptrace/llamaindex/__init__.py

@@ -1,15 +1,16 @@
-
-#pylint: disable=protected-access
+# pylint: disable=protected-access
 import os
-from monocle_apptrace.utils import load_wrapper_from_config
+from monocle_apptrace.utils import get_wrapper_methods_config
+
 
 def get_llm_span_name_for_openai(instance):
     if (hasattr(instance, "_is_azure_client")
-        and callable(getattr(instance, "_is_azure_client"))
-        and instance._is_azure_client()):
+            and callable(getattr(instance, "_is_azure_client"))
+            and instance._is_azure_client()):
         return "llamaindex.azure_openai"
     return "llamaindex.openai"
 
 parent_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
-LLAMAINDEX_METHODS = load_wrapper_from_config(
-    os.path.join(parent_dir, 'metamodel', 'maps', 'llama_index_methods.json'))
+LLAMAINDEX_METHODS = get_wrapper_methods_config(
+    wrapper_methods_config_path=os.path.join(parent_dir, 'metamodel', 'maps', 'llamaindex_methods.json'),
+    attributes_config_base_path=os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

monocle_apptrace/message_processing.py

@@ -0,0 +1,80 @@
+"""
+This module provides utility functions for extracting system, user,
+and assistant messages from various input formats.
+"""
+
+import logging
+from monocle_apptrace.utils import get_attribute
+DATA_INPUT_KEY = "data.input"
+
+logger = logging.getLogger(__name__)
+def extract_messages(args):
+    """Extract system and user messages"""
+    try:
+        messages = []
+        args_input = get_attribute(DATA_INPUT_KEY)
+        if args_input:
+            messages.append(args_input)
+            return messages
+        if args and isinstance(args, tuple) and len(args) > 0:
+            if hasattr(args[0], "messages") and isinstance(args[0].messages, list):
+                for msg in args[0].messages:
+                    if hasattr(msg, 'content') and hasattr(msg, 'type'):
+                        messages.append({msg.type: msg.content})
+            elif isinstance(args[0], list):  #llama
+                for msg in args[0]:
+                    if hasattr(msg, 'content') and hasattr(msg, 'role'):
+                        if hasattr(msg.role, 'value'):
+                            role = msg.role.value
+                        else:
+                            role = msg.role
+                        if msg.role == "system":
+                            messages.append({role: msg.content})
+                        elif msg.role in ["user", "human"]:
+                            user_message = extract_query_from_content(msg.content)
+                            messages.append({role: user_message})
+        return messages
+    except Exception as e:
+        logger.warning("Warning: Error occurred in extract_messages: %s", str(e))
+        return []
+
+
+def extract_assistant_message(response):
+    try:
+        if isinstance(response, str):
+            return [response]
+        if hasattr(response, "content"):
+            return [response.content]
+        if hasattr(response, "message") and hasattr(response.message, "content"):
+            return [response.message.content]
+        if "replies" in response:
+            reply = response["replies"][0]
+            if hasattr(reply, 'content'):
+                return [reply.content]
+            return [reply]
+        if isinstance(response, dict):
+            return [response]
+        return []
+    except Exception as e:
+        logger.warning("Warning: Error occurred in extract_assistant_message: %s", str(e))
+        return []
+
+
+def extract_query_from_content(content):
+    try:
+        query_prefix = "Query:"
+        answer_prefix = "Answer:"
+        query_start = content.find(query_prefix)
+        if query_start == -1:
+            return None
+
+        query_start += len(query_prefix)
+        answer_start = content.find(answer_prefix, query_start)
+        if answer_start == -1:
+            query = content[query_start:].strip()
+        else:
+            query = content[query_start:answer_start].strip()
+        return query
+    except Exception as e:
+        logger.warning("Warning: Error occurred in extract_query_from_content: %s", str(e))
+        return ""

monocle_apptrace/metamodel/entities/README.md

@@ -1,33 +1,44 @@
 # Monocle Entities
 The entity type defines the type of GenAI component that Monocle understand. The monocle instrumentation can extract the relevenat information for this entity. There are a fixed set of [entity types](./entity_types.py) that are defined by Monocle out of the box, eg workflow, model etc. As the GenAI landscape evolves, the Monocle community will introduce a new entity type if the current entities won't represent a new technology component.
 
-## Entity Types
+## Entities
 Following attributes are supported for all entities
 | Name | Description | Required |
 | - | - | - |
 | name | Entity name generated by Monocle | Required |
-| type | Monocle Entity type | True |
+| type | entity.category.type | True |
 
 ### MonocleEntity.Workflow
-Workflow ie the core application code. Supported types are -
+Workflow ie the core application code. 
+#### Workflow types
 - generic
 - langchain
-- llama_index
+- llamaindex
 - haystack
 
 ### MonocleEntity.Model
-GenAI models. Supported types are -
+GenAI models. It has following sub categories
 - generic
 - llm
 - embedding
+#### Model types
+- gpt3
+- gpt-3.5
+- gpt-3.5-turbo
+- gpt4
+- gpt4o
+- gpt4o-mini
+- ollama
+
 Following attributes are supported for all model type entities
 | Name | Description | Required |
 | - | - | - |
-| model_name | Name of model | True |
-
+| parameters | model parameters | false |
+| algorithm | model algorithm | false |
 
 ### MonocleEntity.AppHosting
-Application host services where the workflow code is run. Supported types are -
+Application host services where the workflow code is run.
+#### AppHosting types
 - generic
 - aws_lambda
 - aws_sagemaker
@@ -36,7 +47,12 @@ Application host services where the workflow code is run. Supported types are -
 - azure_mlw
 
 ### MonocleEntity.Inference
-The model hosting infrastructure services. Supported types are -
+The model hosting infrastructure services.
+Following attributes are supported for inference type entities
+| Name | Description | Required |
+| - | - | - |
+| inference_endpoint | connection endpoint for inference service | false |
+#### Inference types
 - generic
 - nvidia_triton
 - openai
@@ -45,8 +61,15 @@ The model hosting infrastructure services. Supported types are -
 - aws_bedrock
 - hugging_face
 
+Following attributes are supported for inference type entities
+| Name | Description | Inference type | Required |
+| - | - | - | - |
+| deployment | model deployment name | azure_oai | false |
+
+
 ### MonocleEntity.VectorStore
-Vector search data stores. Supported types are -
+Vector search data stores.
+#### VectorStore types
 - generic
 - chroma
 - aws_es

monocle_apptrace/metamodel/entities/app_hosting_types.json

@@ -0,0 +1,29 @@
+{
+    "entity" : "app_hosting",
+    "types": [
+        {
+            "name": "aws_lambda",
+            "attributes" : []
+        },
+        {
+            "name": "aws_sagemaker",
+            "attributes" : []
+        },
+        {
+            "name": "azure_func",
+            "attributes" : []
+        },
+        {
+            "name": "azure_mlw",
+            "attributes" : []
+        },
+        {
+            "name": "github_codespace",
+            "attributes" : []
+        },
+        {
+            "name": "generic",
+            "attributes" : []
+        }
+    ]
+}

monocle_apptrace/metamodel/entities/entities.json

@@ -0,0 +1,49 @@
+{
+    "description": "Monocle entities represents kinds GenAI technology components and their implementations supported by Monocle",
+    "monocle_entities": [
+        {
+            "name": "workflow",
+            "attributes" : [],
+            "categories": []
+        },
+        {
+            "name": "model",
+            "attributes" : [
+                {
+                    "attribute_name": "model_name",
+                    "attribute_description": "Model name",
+                    "required": true
+                }
+            ],
+            "categories": [
+                {
+                    "name": "llm",
+                    "attributes" : []
+                },
+                {
+                    "name": "embedding",
+                    "attributes" : []
+                },
+                {
+                    "name": "generic",
+                    "attributes" : []
+                }
+            ]
+        },
+        {
+            "name": "vector_store",
+            "attributes" : [],
+            "categories": []
+        },
+        {
+            "name": "app_hosting",
+            "attributes" : [],
+            "categories": []
+        },
+        {
+            "name": "inference",
+            "attributes" : [],
+            "categories": []
+        }
+    ]
+}

monocle_apptrace/metamodel/entities/inference_types.json

@@ -0,0 +1,33 @@
+{
+    "entity" : "vector_store",
+    "types": [
+        {
+            "name": "aws_sagemaker",
+            "attributes" : []
+        },
+        {
+            "name": "aws_bedrock",
+            "attributes" : []
+        },
+        {
+            "name": "azure_oai",
+            "attributes" : []
+        },
+        {
+            "name": "openai",
+            "attributes" : []
+        },
+        {
+            "name": "nvidia_triton",
+            "attributes" : []
+        },
+        {
+            "name": "hugging_face",
+            "attributes" : []
+        },
+        {
+            "name": "generic",
+            "attributes" : []
+        }
+    ]
+}

monocle_apptrace/metamodel/entities/model_types.json

@@ -0,0 +1,41 @@
+{
+    "entity" : "model",
+    "types": [
+        {
+            "name": "gpt3",
+            "attributes" : []
+        },
+        {
+            "name": "gpt-3.5",
+            "attributes" : []
+        },
+        {
+            "name": "gpt-3.5-turbo",
+            "attributes" : []
+        },
+        {
+            "name": "gpt-3.5",
+            "attributes" : []
+        },
+        {
+            "name": "gpt4",
+            "attributes" : []
+        },
+        {
+            "name": "gpt4o",
+            "attributes" : []
+        },
+        {
+            "name": "gpt4o-mini",
+            "attributes" : []
+        },
+        {
+            "name": "ollama",
+            "attributes" : []
+        },
+        {
+            "name": "generic",
+            "attributes" : []
+        }
+    ]
+}

monocle_apptrace/metamodel/entities/vector_store_types.json

@@ -0,0 +1,25 @@
+{
+    "entity" : "vector_store",
+    "types": [
+        {
+            "name": "chroma",
+            "attributes" : []
+        },
+        {
+            "name": "aws_es",
+            "attributes" : []
+        },
+        {
+            "name": "milvus",
+            "attributes" : []
+        },
+        {
+            "name": "pinecone",
+            "attributes" : []
+        },
+        {
+            "name": "generic",
+            "attributes" : []
+        }
+    ]
+}

monocle_apptrace/metamodel/entities/workflow_types.json

@@ -0,0 +1,22 @@
+{
+    "entity" : "workflow",
+    "types": [
+        {
+            "name": "llama_index",
+            "attributes" : []
+        },
+        {
+            "name": "langchain",
+            "attributes" : []
+        },
+        {
+            "name": "haystack",
+            "attributes" : []
+        },
+        {
+            "name": "generic",
+            "attributes" : []
+        }
+
+    ]
+}

monocle_apptrace/metamodel/maps/attributes/inference/botocore_entities.json

@@ -0,0 +1,27 @@
+{
+  "type": "inference",
+  "attributes": [
+    [
+      {
+        "_comment": "provider type  , inference_endpoint",
+        "attribute": "type",
+        "accessor": "lambda arguments:'inference.aws_sagemaker'"
+      },
+      {
+        "attribute": "inference_endpoint",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['azure_endpoint', 'api_base']) or arguments['instance'].meta.endpoint_url"
+      }
+    ],
+    [
+      {
+        "_comment": "LLM Model",
+        "attribute": "name",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name']) or arguments['kwargs'].get('EndpointName', '')"
+      },
+      {
+        "attribute": "type",
+        "accessor": "lambda arguments: 'model.llm.' + (resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name']) or arguments['kwargs'].get('EndpointName', ''))"
+      }
+    ]
+  ]
+}

monocle_apptrace/metamodel/maps/attributes/inference/haystack_entities.json

@@ -0,0 +1,57 @@
+{
+  "type": "inference",
+  "attributes": [
+    [
+      {
+        "_comment": "provider type ,name , deployment , inference_endpoint",
+        "attribute": "type",
+        "accessor": "lambda arguments:'inference.azure_oai'"
+      },
+      {
+        "attribute": "provider_name",
+        "accessor": "lambda arguments:arguments['kwargs']['provider_name']"
+      },
+      {
+        "attribute": "deployment",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['engine', 'azure_deployment', 'deployment_name', 'deployment_id', 'deployment'])"
+      },
+      {
+        "attribute": "inference_endpoint",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['azure_endpoint', 'api_base'])  or arguments['kwargs']['inference_endpoint']"
+      }
+    ],
+    [
+      {
+        "_comment": "LLM Model",
+        "attribute": "name",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name'])"
+      },
+      {
+        "attribute": "type",
+        "accessor": "lambda arguments: 'model.llm.'+resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name'])"
+      }
+    ]
+  ],
+  "events": [
+    { "name":"data.input",
+      "attributes": [
+
+        {
+            "_comment": "this is instruction and user query to LLM",
+            "attribute": "input",
+            "accessor": "lambda arguments: extract_messages(arguments['args'])"
+        }
+      ]
+    },
+    {
+      "name":"data.output",
+      "attributes": [
+        {
+            "_comment": "this is response from LLM",
+            "attribute": "response",
+            "accessor": "lambda response: extract_assistant_message(response)"
+        }
+      ]
+   }
+  ]
+}

monocle_apptrace/metamodel/maps/attributes/inference/langchain_entities.json

@@ -0,0 +1,57 @@
+{
+  "type": "inference",
+  "attributes": [
+    [
+      {
+        "_comment": "provider type ,name , deployment , inference_endpoint",
+        "attribute": "type",
+        "accessor": "lambda arguments:'inference.azure_oai'"
+      },
+      {
+        "attribute": "provider_name",
+        "accessor": "lambda arguments:arguments['kwargs']['provider_name']"
+      },
+      {
+        "attribute": "deployment",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['engine', 'azure_deployment', 'deployment_name', 'deployment_id', 'deployment'])"
+      },
+      {
+        "attribute": "inference_endpoint",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['azure_endpoint', 'api_base']) or arguments['kwargs']['inference_endpoint']"
+      }
+    ],
+    [
+      {
+        "_comment": "LLM Model",
+        "attribute": "name",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name']) or arguments['instance'].model_id"
+      },
+      {
+        "attribute": "type",
+        "accessor": "lambda arguments: 'model.llm.'+ (resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name']) or arguments['instance'].model_id)"
+      }
+    ]
+  ],
+  "events": [
+    { "name":"data.input",
+      "attributes": [
+
+        {
+            "_comment": "this is instruction and user query to LLM",
+            "attribute": "input",
+            "accessor": "lambda arguments: extract_messages(arguments['args'])"
+        }
+      ]
+    },
+    {
+      "name":"data.output",
+      "attributes": [
+        {
+            "_comment": "this is response from LLM",
+            "attribute": "response",
+            "accessor": "lambda response: extract_assistant_message(response)"
+        }
+      ]
+   }
+  ]
+}

monocle_apptrace/metamodel/maps/attributes/inference/llamaindex_entities.json

@@ -0,0 +1,57 @@
+{
+  "type": "inference",
+  "attributes": [
+    [
+      {
+        "_comment": "provider type ,name , deployment , inference_endpoint",
+        "attribute": "type",
+        "accessor": "lambda arguments:'inference.azure_oai'"
+      },
+      {
+        "attribute": "provider_name",
+        "accessor": "lambda arguments:arguments['kwargs']['provider_name']"
+      },
+      {
+        "attribute": "deployment",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['engine', 'azure_deployment', 'deployment_name', 'deployment_id', 'deployment'])"
+      },
+      {
+        "attribute": "inference_endpoint",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['azure_endpoint', 'api_base']) or arguments['kwargs']['inference_endpoint']"
+      }
+    ],
+    [
+      {
+        "_comment": "LLM Model",
+        "attribute": "name",
+        "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name'])"
+      },
+      {
+        "attribute": "type",
+        "accessor": "lambda arguments: 'model.llm.'+resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name'])"
+      }
+    ]
+  ],
+  "events": [
+    { "name":"data.input",
+      "attributes": [
+
+        {
+            "_comment": "this is instruction and user query to LLM",
+            "attribute": "input",
+            "accessor": "lambda arguments: extract_messages(arguments['args'])"
+        }
+      ]
+    },
+    {
+      "name":"data.output",
+      "attributes": [
+        {
+            "_comment": "this is response from LLM",
+            "attribute": "response",
+            "accessor": "lambda response: extract_assistant_message(response)"
+        }
+      ]
+   }
+  ]
+}

monocle_apptrace/metamodel/maps/attributes/retrieval/haystack_entities.json

@@ -0,0 +1,31 @@
+{
+  "type": "retrieval",
+  "attributes": [
+    [
+      {
+        "_comment": "vector store name and type",
+        "attribute": "name",
+         "accessor": "lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['document_store', '_document_store']).__class__.__name__"
+      },
+      {
+        "attribute": "type",
+        "accessor": "lambda arguments: 'vectorstore.'+resolve_from_alias(arguments['instance'].__dict__, ['document_store', '_document_store']).__class__.__name__"
+      },
+      {
+        "attribute": "deployment",
+        "accessor": "lambda arguments: get_vectorstore_deployment(resolve_from_alias(arguments['instance'].__dict__, ['document_store', '_document_store']).__dict__)"
+      }
+    ],
+    [
+      {
+        "_comment": "embedding model name and type",
+        "attribute": "name",
+        "accessor": "lambda arguments: get_embedding_model()"
+      },
+      {
+        "attribute": "type",
+        "accessor": "lambda arguments: 'model.embedding.'+get_embedding_model()"
+      }
+    ]
+  ]
+}

monocle_apptrace/metamodel/maps/attributes/retrieval/langchain_entities.json

@@ -0,0 +1,31 @@
+{
+  "type": "retrieval",
+  "attributes": [
+    [
+      {
+        "_comment": "vector store name and type",
+        "attribute": "name",
+        "accessor": "lambda arguments: type(arguments['instance'].vectorstore).__name__"
+      },
+      {
+        "attribute": "type",
+        "accessor": "lambda arguments: 'vectorstore.'+type(arguments['instance'].vectorstore).__name__"
+      },
+      {
+        "attribute": "deployment",
+        "accessor": "lambda arguments: get_vectorstore_deployment(arguments['instance'].vectorstore.__dict__)"
+      }
+    ],
+    [
+      {
+        "_comment": "embedding model name and type",
+        "attribute": "name",
+        "accessor": "lambda arguments: arguments['instance'].vectorstore.embeddings.model"
+      },
+      {
+        "attribute": "type",
+        "accessor": "lambda arguments: 'model.embedding.'+arguments['instance'].vectorstore.embeddings.model"
+      }
+    ]
+  ]
+}