monocle-apptrace 0.3.0b1__py3-none-any.whl → 0.3.0b3__py3-none-any.whl

monocle_apptrace/instrumentation/metamodel/haystack/_helper.py

@@ -0,0 +1,127 @@
+import logging
+from monocle_apptrace.instrumentation.common.utils import (
+    Option,
+    get_keys_as_tuple,
+    get_nested_value,
+    try_option,
+)
+logger = logging.getLogger(__name__)
+
+
+def extract_messages(kwargs):
+    try:
+        messages = []
+        if isinstance(kwargs, dict):
+            if 'system_prompt' in kwargs and kwargs['system_prompt']:
+                system_message = kwargs['system_prompt']
+                messages.append({"system" : system_message})
+            if 'prompt' in kwargs and kwargs['prompt']:
+                user_message = extract_question_from_prompt(kwargs['prompt'])
+                messages.append({"user": user_message})
+        return [str(message) for message in messages]
+    except Exception as e:
+        logger.warning("Warning: Error occurred in extract_messages: %s", str(e))
+        return []
+
+
+def extract_question_from_prompt(content):
+    try:
+        question_prefix = "Question:"
+        answer_prefix = "Answer:"
+
+        question_start = content.find(question_prefix)
+        if question_start == -1:
+            return None  # Return None if "Question:" is not found
+
+        question_start += len(question_prefix)
+        answer_start = content.find(answer_prefix, question_start)
+        if answer_start == -1:
+            question = content[question_start:].strip()
+        else:
+            question = content[question_start:answer_start].strip()
+
+        return question
+    except Exception as e:
+        logger.warning("Warning: Error occurred in extract_question_from_prompt: %s", str(e))
+        return ""
+
+
+def extract_assistant_message(response):
+    try:
+        if "replies" in response:
+            reply = response["replies"][0]
+            if hasattr(reply, 'content'):
+                return [reply.content]
+            return [reply]
+    except Exception as e:
+        logger.warning("Warning: Error occurred in extract_assistant_message: %s", str(e))
+        return []
+
+
+def get_vectorstore_deployment(my_map):
+    if isinstance(my_map, dict):
+        if '_client_settings' in my_map:
+            client = my_map['_client_settings'].__dict__
+            host, port = get_keys_as_tuple(client, 'host', 'port')
+            if host:
+                return f"{host}:{port}" if port else host
+        keys_to_check = ['client', '_client']
+        host = get_host_from_map(my_map, keys_to_check)
+        if host:
+            return host
+    else:
+        if hasattr(my_map, 'client') and '_endpoint' in my_map.client.__dict__:
+            return my_map.client.__dict__['_endpoint']
+        host, port = get_keys_as_tuple(my_map.__dict__, 'host', 'port')
+        if host:
+            return f"{host}:{port}" if port else host
+    return None
+
+
+def get_host_from_map(my_map, keys_to_check):
+    for key in keys_to_check:
+        seed_connections = get_nested_value(my_map, [key, 'transport', 'seed_connections'])
+        if seed_connections and 'host' in seed_connections[0].__dict__:
+            return seed_connections[0].__dict__['host']
+    return None
+
+
+def resolve_from_alias(my_map, alias):
+    """Find a alias that is not none from list of aliases"""
+
+    for i in alias:
+        if i in my_map.keys():
+            return my_map[i]
+    return None
+
+def extract_inference_endpoint(instance):
+    inference_endpoint: Option[str] = try_option(getattr, instance.client, 'base_url').map(str)
+    if inference_endpoint.is_none():
+        inference_endpoint = try_option(getattr, instance.client.meta, 'endpoint_url').map(str)
+
+    return inference_endpoint.unwrap_or(None)
+
+def extract_embeding_model(instance):
+    pipeline = try_option(getattr, instance, '__haystack_added_to_pipeline__')
+    return pipeline.map(lambda p: try_option(getattr, p, 'get_component').map(
+        lambda g: try_option(getattr, g('text_embedder'), 'model').unwrap_or(None)).unwrap_or(None)).unwrap_or(None)
+
+def update_span_from_llm_response(response, instance):
+    meta_dict = {}
+    if response is not None and isinstance(response, dict) and "meta" in response:
+        token_usage = response["meta"][0]["usage"]
+        if token_usage is not None:
+            temperature = instance.__dict__.get("temperature", None)
+            meta_dict.update({"temperature": temperature})
+            meta_dict.update(
+                {"completion_tokens": token_usage.get("completion_tokens") or token_usage.get("output_tokens")})
+            meta_dict.update({"prompt_tokens": token_usage.get("prompt_tokens") or token_usage.get("input_tokens")})
+            meta_dict.update({"total_tokens": token_usage.get("total_tokens")})
+    return meta_dict
+
+
+def update_output_span_events(results):
+    output_arg_text = " ".join([doc.content for doc in results['documents']])
+    if len(output_arg_text) > 100:
+        output_arg_text = output_arg_text[:100] + "..."
+    return output_arg_text

monocle_apptrace/instrumentation/metamodel/haystack/entities/inference.py

@@ -0,0 +1,76 @@
+from monocle_apptrace.instrumentation.metamodel.haystack import (
+    _helper,
+)
+
+INFERENCE = {
+    "type": "inference",
+    "attributes": [
+        [
+            {
+                "_comment": "provider type ,name , deployment , inference_endpoint",
+                "attribute": "type",
+                "accessor": lambda arguments: 'inference.azure_oai'
+            },
+            {
+                "attribute": "provider_name",
+                "accessor": lambda arguments: arguments['kwargs']['provider_name']
+            },
+            {
+                "attribute": "deployment",
+                "accessor": lambda arguments: _helper.resolve_from_alias(arguments['instance'].__dict__,
+                                                                         ['engine', 'azure_deployment',
+                                                                          'deployment_name', 'deployment_id',
+                                                                          'deployment'])
+            },
+            {
+                "attribute": "inference_endpoint",
+                "accessor": lambda arguments: _helper.resolve_from_alias(arguments['instance'].__dict__, ['api_base_url']) or _helper.extract_inference_endpoint(arguments['instance'])
+            }
+        ],
+        [
+            {
+                "_comment": "LLM Model",
+                "attribute": "name",
+                "accessor": lambda arguments: _helper.resolve_from_alias(arguments['instance'].__dict__,
+                                                                         ['model', 'model_name'])
+            },
+            {
+                "attribute": "type",
+                "accessor": lambda arguments: 'model.llm.' + _helper.resolve_from_alias(arguments['instance'].__dict__,
+                                                                                        ['model', 'model_name'])
+            }
+        ]
+    ],
+    "events": [
+        {"name": "data.input",
+         "attributes": [
+
+             {
+                 "_comment": "this is instruction and user query to LLM",
+                 "attribute": "input",
+                 "accessor": lambda arguments: _helper.extract_messages(arguments['kwargs'])
+             }
+         ]
+         },
+        {
+            "name": "data.output",
+            "attributes": [
+                {
+                    "_comment": "this is response from LLM",
+                    "attribute": "response",
+                    "accessor": lambda arguments: _helper.extract_assistant_message(arguments['result'])
+                }
+            ]
+        },
+        {
+            "name": "metadata",
+            "attributes": [
+                {
+                    "_comment": "this is metadata usage from LLM",
+                    "accessor": lambda arguments: _helper.update_span_from_llm_response(arguments['result'],
+                                                                                        arguments['instance'])
+                }
+            ]
+        }
+    ]
+}

monocle_apptrace/instrumentation/metamodel/haystack/entities/retrieval.py

@@ -0,0 +1,61 @@
+from monocle_apptrace.instrumentation.metamodel.haystack import (_helper, )
+from monocle_apptrace.instrumentation.common.utils import get_attribute
+
+RETRIEVAL = {
+    "type": "retrieval",
+    "attributes": [
+        [
+            {
+                "_comment": "vector store name and type",
+                "attribute": "name",
+                "accessor": lambda arguments: _helper.resolve_from_alias(arguments['instance'].__dict__,
+                                                                         ['document_store',
+                                                                          '_document_store']).__class__.__name__
+            },
+            {
+                "attribute": "type",
+                "accessor": lambda arguments: 'vectorstore.' + _helper.resolve_from_alias(
+                    arguments['instance'].__dict__, ['document_store', '_document_store']).__class__.__name__
+            },
+            {
+                "attribute": "deployment",
+                "accessor": lambda arguments: _helper.get_vectorstore_deployment(
+                    _helper.resolve_from_alias(arguments['instance'].__dict__,
+                                               ['document_store', '_document_store']).__dict__)
+            }
+        ],
+        [
+            {
+                "_comment": "embedding model name and type",
+                "attribute": "name",
+                "accessor": lambda arguments: _helper.extract_embeding_model(arguments['instance'])
+            },
+            {
+                "attribute": "type",
+                "accessor": lambda arguments: 'model.embedding.' + _helper.extract_embeding_model(arguments['instance'])
+            }
+        ]
+    ],
+    "events": [
+        {"name": "data.input",
+         "attributes": [
+
+             {
+                 "_comment": "this is instruction and user query to LLM",
+                 "attribute": "input",
+                 "accessor": lambda arguments: get_attribute("input")
+             }
+         ]
+         },
+        {
+            "name": "data.output",
+            "attributes": [
+                {
+                    "_comment": "this is result from LLM",
+                    "attribute": "response",
+                    "accessor": lambda arguments: _helper.update_output_span_events(arguments['result'])
+                }
+            ]
+        }
+    ]
+}

monocle_apptrace/instrumentation/metamodel/haystack/methods.py

@@ -0,0 +1,42 @@
+from monocle_apptrace.instrumentation.common.wrapper import task_wrapper
+from monocle_apptrace.instrumentation.metamodel.haystack.entities.inference import INFERENCE
+from monocle_apptrace.instrumentation.metamodel.haystack.entities.retrieval import RETRIEVAL
+
+HAYSTACK_METHODS = [
+    {
+        "package": "haystack.components.retrievers.in_memory",
+        "object": "InMemoryEmbeddingRetriever",
+        "method": "run",
+        "span_name": "haystack.retriever",
+        "wrapper_method": task_wrapper,
+        "output_processor": RETRIEVAL
+    },
+    {
+        "package": "haystack_integrations.components.retrievers.opensearch",
+        "object": "OpenSearchEmbeddingRetriever",
+        "method": "run",
+        "span_name": "haystack.retriever",
+        "wrapper_method": task_wrapper,
+        "output_processor": RETRIEVAL
+    },
+    {
+        "package": "haystack.components.generators.openai",
+        "object": "OpenAIGenerator",
+        "method": "run",
+        "wrapper_method": task_wrapper,
+        "output_processor": INFERENCE
+    },
+    {
+        "package": "haystack.components.generators.chat.openai",
+        "object": "OpenAIChatGenerator",
+        "method": "run",
+        "wrapper_method": task_wrapper,
+        "output_processor": INFERENCE
+    },
+    {
+        "package": "haystack.core.pipeline.pipeline",
+        "object": "Pipeline",
+        "method": "run",
+        "wrapper_method": task_wrapper
+    }
+]

monocle_apptrace/instrumentation/metamodel/langchain/_helper.py

@@ -0,0 +1,121 @@
+"""
+This module provides utility functions for extracting system, user,
+and assistant messages from various input formats.
+"""
+
+import logging
+from monocle_apptrace.instrumentation.common.utils import (
+    Option,
+    get_keys_as_tuple,
+    get_nested_value,
+    try_option,
+)
+
+
+logger = logging.getLogger(__name__)
+
+
+def extract_messages(args):
+    """Extract system and user messages"""
+    try:
+        messages = []
+        if args and isinstance(args, (list, tuple)) and len(args) > 0:
+            if hasattr(args[0], "messages") and isinstance(args[0].messages, list):
+                for msg in args[0].messages:
+                    if hasattr(msg, 'content') and hasattr(msg, 'type'):
+                        messages.append({msg.type: msg.content})
+        return [str(d) for d in messages]
+    except Exception as e:
+        logger.warning("Warning: Error occurred in extract_messages: %s", str(e))
+        return []
+
+
+def extract_assistant_message(response):
+    try:
+        if isinstance(response, str):
+            return [response]
+        if hasattr(response, "content"):
+            return [response.content]
+        if hasattr(response, "message") and hasattr(response.message, "content"):
+            return [response.message.content]
+    except Exception as e:
+        logger.warning("Warning: Error occurred in extract_assistant_message: %s", str(e))
+        return []
+
+
+def extract_provider_name(instance):
+    provider_url: Option[str] = try_option(getattr, instance.client._client.base_url, 'host')
+    return provider_url.unwrap_or(None)
+
+
+def extract_inference_endpoint(instance):
+    inference_endpoint: Option[str] = try_option(getattr, instance.client._client, 'base_url').map(str)
+    if inference_endpoint.is_none() and "meta" in instance.client.__dict__:
+        inference_endpoint = try_option(getattr, instance.client.meta, 'endpoint_url').map(str)
+
+    return inference_endpoint.unwrap_or(extract_provider_name(instance))
+
+
+def extract_vectorstore_deployment(my_map):
+    if isinstance(my_map, dict):
+        if '_client_settings' in my_map:
+            client = my_map['_client_settings'].__dict__
+            host, port = get_keys_as_tuple(client, 'host', 'port')
+            if host:
+                return f"{host}:{port}" if port else host
+        keys_to_check = ['client', '_client']
+        host = __get_host_from_map(my_map, keys_to_check)
+        if host:
+            return host
+    else:
+        if hasattr(my_map, 'client') and '_endpoint' in my_map.client.__dict__:
+            return my_map.client.__dict__['_endpoint']
+        host, port = get_keys_as_tuple(my_map.__dict__, 'host', 'port')
+        if host:
+            return f"{host}:{port}" if port else host
+    return None
+
+def __get_host_from_map(my_map, keys_to_check):
+    for key in keys_to_check:
+        seed_connections = get_nested_value(my_map, [key, 'transport', 'seed_connections'])
+        if seed_connections and 'host' in seed_connections[0].__dict__:
+            return seed_connections[0].__dict__['host']
+    return None
+
+def resolve_from_alias(my_map, alias):
+    """Find a alias that is not none from list of aliases"""
+
+    for i in alias:
+        if i in my_map.keys():
+            return my_map[i]
+    return None
+
+
+def update_input_span_events(args):
+    return args[0] if len(args) > 0 else ""
+
+
+def update_output_span_events(results):
+    output_arg_text = " ".join([doc.page_content for doc in results if hasattr(doc, 'page_content')])
+    if len(output_arg_text) > 100:
+        output_arg_text = output_arg_text[:100] + "..."
+    return output_arg_text
+
+
+def update_span_from_llm_response(response, instance):
+    meta_dict = {}
+    if response is not None and hasattr(response, "response_metadata"):
+        if hasattr(response, "usage_metadata") and response.usage_metadata is not None:
+            token_usage = response.usage_metadata
+        else:
+            response_metadata = response.response_metadata
+            token_usage = response_metadata.get("token_usage")
+        if token_usage is not None:
+            temperature = instance.__dict__.get("temperature", None)
+            meta_dict.update({"temperature": temperature})
+            meta_dict.update(
+                {"completion_tokens": token_usage.get("completion_tokens") or token_usage.get("output_tokens")})
+            meta_dict.update({"prompt_tokens": token_usage.get("prompt_tokens") or token_usage.get("input_tokens")})
+            meta_dict.update({"total_tokens": token_usage.get("total_tokens")})
+    return meta_dict
+

monocle_apptrace/instrumentation/metamodel/langchain/entities/inference.py

@@ -0,0 +1,71 @@
+from monocle_apptrace.instrumentation.metamodel.langchain import (
+    _helper,
+)
+from monocle_apptrace.instrumentation.common.utils import resolve_from_alias
+
+INFERENCE = {
+    "type": "inference",
+    "attributes": [
+        [
+            {
+                "_comment": "provider type ,name , deployment , inference_endpoint",
+                "attribute": "type",
+                "accessor": lambda arguments: 'inference.azure_oai'
+            },
+            {
+                "attribute": "provider_name",
+                "accessor": lambda arguments: _helper.extract_provider_name(arguments['instance'])
+            },
+            {
+                "attribute": "deployment",
+                "accessor": lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['engine', 'azure_deployment', 'deployment_name', 'deployment_id', 'deployment'])
+            },
+            {
+                "attribute": "inference_endpoint",
+                "accessor": lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['azure_endpoint', 'api_base', 'endpoint']) or _helper.extract_inference_endpoint(arguments['instance'])
+            }
+        ],
+        [
+            {
+                "_comment": "LLM Model",
+                "attribute": "name",
+                "accessor": lambda arguments: resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name', 'endpoint_name', 'deployment_name'])
+            },
+            {
+                "attribute": "type",
+                "accessor": lambda arguments: 'model.llm.' + resolve_from_alias(arguments['instance'].__dict__, ['model', 'model_name', 'endpoint_name', 'deployment_name'])
+            }
+        ]
+    ],
+    "events": [
+        {"name": "data.input",
+         "attributes": [
+
+             {
+                 "_comment": "this is instruction and user query to LLM",
+                 "attribute": "input",
+                 "accessor": lambda arguments: _helper.extract_messages(arguments['args'])
+             }
+         ]
+         },
+        {
+            "name": "data.output",
+            "attributes": [
+                {
+                    "_comment": "this is result from LLM",
+                    "attribute": "response",
+                    "accessor": lambda arguments: _helper.extract_assistant_message(arguments['result'])
+                }
+            ]
+        },
+        {
+            "name": "metadata",
+            "attributes": [
+                {
+                    "_comment": "this is metadata usage from LLM",
+                    "accessor": lambda arguments: _helper.update_span_from_llm_response(arguments['result'], arguments['instance'])
+                }
+            ]
+        }
+    ]
+}

monocle_apptrace/instrumentation/metamodel/langchain/entities/retrieval.py

@@ -0,0 +1,58 @@
+from monocle_apptrace.instrumentation.metamodel.langchain import (
+    _helper,
+)
+
+RETRIEVAL = {
+    "type": "retrieval",
+    "attributes": [
+        [
+            {
+                "_comment": "vector store name and type",
+                "attribute": "name",
+                "accessor": lambda arguments: type(arguments['instance'].vectorstore).__name__
+            },
+            {
+                "attribute": "type",
+                "accessor": lambda arguments: 'vectorstore.' + type(arguments['instance'].vectorstore).__name__
+            },
+            {
+                "attribute": "deployment",
+                "accessor": lambda arguments: _helper.extract_vectorstore_deployment(
+                    arguments['instance'].vectorstore.__dict__)
+            }
+        ],
+        [
+            {
+                "_comment": "embedding model name and type",
+                "attribute": "name",
+                "accessor": lambda arguments: _helper.resolve_from_alias(arguments['instance'].vectorstore.embeddings.__dict__,['endpoint_name','model_id','model'])
+            },
+            {
+                "attribute": "type",
+                "accessor": lambda arguments: 'model.embedding.' + _helper.resolve_from_alias(arguments['instance'].vectorstore.embeddings.__dict__,['endpoint_name','model_id','model'])
+            }
+        ]
+    ],
+    "events": [
+        {"name": "data.input",
+         "attributes": [
+
+             {
+                 "_comment": "this is instruction and user query to LLM",
+                 "attribute": "input",
+                 "accessor": lambda arguments: _helper.update_input_span_events(arguments['args'])
+             }
+         ]
+         },
+        {
+            "name": "data.output",
+            "attributes": [
+                {
+                    "_comment": "this is result from LLM",
+                    "attribute": "response",
+                    "accessor": lambda arguments: _helper.update_output_span_events(arguments['result'])
+                }
+            ]
+        }
+    ]
+}

monocle_apptrace/instrumentation/metamodel/langchain/methods.py

@@ -0,0 +1,105 @@
+from monocle_apptrace.instrumentation.common.wrapper import atask_wrapper, task_wrapper
+from monocle_apptrace.instrumentation.metamodel.langchain.entities.inference import (
+    INFERENCE,
+)
+from monocle_apptrace.instrumentation.metamodel.langchain.entities.retrieval import (
+    RETRIEVAL,
+)
+
+LANGCHAIN_METHODS = [
+    {
+        "package": "langchain.prompts.base",
+        "object": "BasePromptTemplate",
+        "method": "invoke",
+        "wrapper_method": task_wrapper
+    },
+    {
+        "package": "langchain.prompts.base",
+        "object": "BasePromptTemplate",
+        "method": "ainvoke",
+        "wrapper_method": atask_wrapper
+    },
+    {
+        "package": "langchain.chat_models.base",
+        "object": "BaseChatModel",
+        "method": "invoke",
+        "wrapper_method": task_wrapper,
+        "output_processor": INFERENCE
+    },
+    {
+        "package": "langchain.chat_models.base",
+        "object": "BaseChatModel",
+        "method": "ainvoke",
+        "wrapper_method": atask_wrapper,
+        "output_processor": INFERENCE
+    },
+    {
+        "package": "langchain_core.language_models.llms",
+        "object": "LLM",
+        "method": "_generate",
+        "wrapper_method": task_wrapper,
+        "output_processor": INFERENCE
+    },
+    {
+        "package": "langchain_core.language_models.llms",
+        "object": "LLM",
+        "method": "_agenerate",
+        "wrapper_method": atask_wrapper,
+        "output_processor": INFERENCE
+    },
+    {
+        "package": "langchain_core.language_models.llms",
+        "object": "BaseLLM",
+        "method": "invoke",
+        "wrapper_method": task_wrapper,
+        "output_processor": INFERENCE
+    },
+    {
+        "package": "langchain_core.language_models.llms",
+        "object": "BaseLLM",
+        "method": "ainvoke",
+        "wrapper_method": atask_wrapper,
+        "output_processor": INFERENCE
+    },
+    {
+        "package": "langchain_core.retrievers",
+        "object": "BaseRetriever",
+        "method": "invoke",
+        "wrapper_method": task_wrapper,
+        "output_processor": RETRIEVAL
+
+    },
+    {
+        "package": "langchain_core.retrievers",
+        "object": "BaseRetriever",
+        "method": "ainvoke",
+        "wrapper_method": atask_wrapper,
+        "output_processor": RETRIEVAL
+    },
+    {
+        "package": "langchain.schema",
+        "object": "BaseOutputParser",
+        "method": "invoke",
+        "wrapper_method": task_wrapper
+    },
+    {
+        "package": "langchain.schema",
+        "object": "BaseOutputParser",
+        "method": "ainvoke",
+        "wrapper_method": atask_wrapper
+    },
+    {
+        "package": "langchain.schema.runnable",
+        "object": "RunnableSequence",
+        "method": "invoke",
+        "span_name": "langchain.workflow",
+        "wrapper_method": task_wrapper
+    },
+    {
+        "package": "langchain.schema.runnable",
+        "object": "RunnableSequence",
+        "method": "ainvoke",
+        "span_name": "langchain.workflow",
+        "wrapper_method": atask_wrapper
+    }
+]