prompty 0.1.10__py3-none-any.whl → 0.1.33__py3-none-any.whl

prompty/tracer.py

@@ -1,6 +1,9 @@
 import os
 import json
 import inspect
+import numbers
+import traceback
+import importlib
 import contextlib
 from pathlib import Path
 from numbers import Number
@@ -10,6 +13,18 @@ from functools import wraps, partial
 from typing import Any, Callable, Dict, Iterator, List
 
 
+# clean up key value pairs for sensitive values
+def sanitize(key: str, value: Any) -> Any:
+    if isinstance(value, str) and any(
+        [s in key.lower() for s in ["key", "token", "secret", "password", "credential"]]
+    ):
+        return len(str(value)) * "*"
+    elif isinstance(value, dict):
+        return {k: sanitize(k, v) for k, v in value.items()}
+    else:
+        return value
+
+
 class Tracer:
     _tracers: Dict[str, Callable[[str], Iterator[Callable[[str, Any], None]]]] = {}
 
@@ -30,7 +45,11 @@ class Tracer:
             traces = [
                 stack.enter_context(tracer(name)) for tracer in cls._tracers.values()
             ]
-            yield lambda key, value: [trace(key, value) for trace in traces]
+            yield lambda key, value: [
+                # normalize and sanitize any trace values
+                trace(key, sanitize(key, to_dict(value)))
+                for trace in traces
+            ]
 
 
 def to_dict(obj: Any) -> Dict[str, Any]:
@@ -69,11 +88,14 @@ def _name(func: Callable, args):
     else:
         signature = f"{func.__module__}.{func.__name__}"
 
-    # core invoker gets special treatment
-    core_invoker = signature == "prompty.core.Invoker.__call__"
+    # core invoker gets special treatment prompty.invoker.Invoker
+    core_invoker = signature.startswith("prompty.invoker.Invoker.run")
     if core_invoker:
         name = type(args[0]).__name__
-        signature = f"{args[0].__module__}.{args[0].__class__.__name__}.invoke"
+        if signature.endswith("async"):
+            signature = f"{args[0].__module__}.{args[0].__class__.__name__}.invoke_async"
+        else:
+            signature = f"{args[0].__module__}.{args[0].__class__.__name__}.invoke"
     else:
         name = func.__name__
 
@@ -93,7 +115,9 @@ def _results(result: Any) -> dict:
     return to_dict(result) if result is not None else "None"
 
 
-def _trace_sync(func: Callable = None, *, description: str = None) -> Callable:
+def _trace_sync(
+    func: Callable = None, *, description: str = None, itemtype: str = None
+) -> Callable:
     description = description or ""
 
     @wraps(func)
@@ -104,18 +128,41 @@ def _trace_sync(func: Callable = None, *, description: str = None) -> Callable:
             if description and description != "":
                 trace("description", description)
 
+            if itemtype and itemtype != "":
+                trace("type", itemtype)
+
             inputs = _inputs(func, args, kwargs)
             trace("inputs", inputs)
 
-            result = func(*args, **kwargs)
-            trace("result", _results(result))
+            try:
+                result = func(*args, **kwargs)
+                trace("result", _results(result))
+            except Exception as e:
+                trace(
+                    "result",
+                    {
+                        "exception": {
+                            "type": type(e),
+                            "traceback": (
+                                traceback.format_tb(tb=e.__traceback__)
+                                if e.__traceback__
+                                else None
+                            ),
+                            "message": str(e),
+                            "args": to_dict(e.args),
+                        }
+                    },
+                )
+                raise e
 
             return result
 
     return wrapper
 
 
-def _trace_async(func: Callable = None, *, description: str = None) -> Callable:
+def _trace_async(
+    func: Callable = None, *, description: str = None, itemtype: str = None
+) -> Callable:
     description = description or ""
 
     @wraps(func)
@@ -126,24 +173,46 @@ def _trace_async(func: Callable = None, *, description: str = None) -> Callable:
             if description and description != "":
                 trace("description", description)
 
+            if itemtype and itemtype != "":
+                trace("type", itemtype)
+
             inputs = _inputs(func, args, kwargs)
             trace("inputs", inputs)
-
-            result = await func(*args, **kwargs)
-            trace("result", _results(result))
+            try:
+                result = await func(*args, **kwargs)
+                trace("result", _results(result))
+            except Exception as e:
+                trace(
+                    "result",
+                    {
+                        "exception": {
+                            "type": type(e),
+                            "traceback": (
+                                traceback.format_tb(tb=e.__traceback__)
+                                if e.__traceback__
+                                else None
+                            ),
+                            "message": str(e),
+                            "args": to_dict(e.args),
+                        }
+                    },
+                )
+                raise e
 
             return result
 
     return wrapper
 
 
-def trace(func: Callable = None, *, description: str = None) -> Callable:
+def trace(
+    func: Callable = None, *, description: str = None, itemtype: str = None
+) -> Callable:
     if func is None:
-        return partial(trace, description=description)
+        return partial(trace, description=description, itemtype=itemtype)
 
     wrapped_method = _trace_async if inspect.iscoroutinefunction(func) else _trace_sync
 
-    return wrapped_method(func, description=description)
+    return wrapped_method(func, description=description, itemtype=itemtype)
 
 
 class PromptyTracer:
@@ -163,6 +232,9 @@ class PromptyTracer:
         try:
             self.stack.append({"name": name})
             frame = self.stack[-1]
+            frame["__time"] = {
+                "start": datetime.now(),
+            }
 
             def add(key: str, value: Any) -> None:
                 if key not in frame:
@@ -177,26 +249,92 @@ class PromptyTracer:
             yield add
         finally:
             frame = self.stack.pop()
+            start: datetime = frame["__time"]["start"]
+            end: datetime = datetime.now()
+
+            # add duration to frame
+            frame["__time"] = {
+                "start": start.strftime("%Y-%m-%dT%H:%M:%S.%f"),
+                "end": end.strftime("%Y-%m-%dT%H:%M:%S.%f"),
+                "duration": int((end - start).total_seconds() * 1000),
+            }
+
+            # hoist usage to parent frame
+            if "result" in frame and isinstance(frame["result"], dict):
+                if "usage" in frame["result"]:
+                    frame["__usage"] = self.hoist_item(
+                        frame["result"]["usage"],
+                        frame["__usage"] if "__usage" in frame else {},
+                    )
+
+            # streamed results may have usage as well
+            if "result" in frame and isinstance(frame["result"], list):
+                for result in frame["result"]:
+                    if (
+                        isinstance(result, dict)
+                        and "usage" in result
+                        and isinstance(result["usage"], dict)
+                    ):
+                        frame["__usage"] = self.hoist_item(
+                            result["usage"],
+                            frame["__usage"] if "__usage" in frame else {},
+                        )
+
+            # add any usage frames from below
+            if "__frames" in frame:
+                for child in frame["__frames"]:
+                    if "__usage" in child:
+                        frame["__usage"] = self.hoist_item(
+                            child["__usage"],
+                            frame["__usage"] if "__usage" in frame else {},
+                        )
+
             # if stack is empty, dump the frame
             if len(self.stack) == 0:
-                trace_file = (
-                    self.output
-                    / f"{frame['name']}.{datetime.now().strftime('%Y%m%d.%H%M%S')}.ptrace"
-                )
-
-                with open(trace_file, "w") as f:
-                    json.dump(frame, f, indent=4)
+                self.write_trace(frame)
             # otherwise, append the frame to the parent
             else:
                 if "__frames" not in self.stack[-1]:
                     self.stack[-1]["__frames"] = []
                 self.stack[-1]["__frames"].append(frame)
 
+    def hoist_item(self, src: Dict[str, Any], cur: Dict[str, Any]) -> None:
+        for key, value in src.items():
+            if value is None or isinstance(value, list) or isinstance(value, dict):
+                continue
+            try:
+                if key not in cur:
+                    cur[key] = value
+                else:
+                    cur[key] += value
+            except:
+                continue
+
+        return cur
+
+    def write_trace(self, frame: Dict[str, Any]) -> None:
+        trace_file = (
+            self.output
+            / f"{frame['name']}.{datetime.now().strftime('%Y%m%d.%H%M%S')}.tracy"
+        )
+
+        v = importlib.metadata.version("prompty")
+        enriched_frame = {
+            "runtime": "python",
+            "version": v,
+            "trace": frame,
+        }
+
+        with open(trace_file, "w") as f:
+            json.dump(enriched_frame, f, indent=4)
+
 
 @contextlib.contextmanager
 def console_tracer(name: str) -> Iterator[Callable[[str, Any], None]]:
     try:
         print(f"Starting {name}")
-        yield lambda key, value: print(f"{key}:\n{json.dumps(value, indent=4)}")
+        yield lambda key, value: print(
+            f"{key}:\n{json.dumps(to_dict(value), indent=4)}"
+        )
     finally:
         print(f"Ending {name}")

prompty/utils.py

@@ -0,0 +1,105 @@
+import re
+import yaml
+import json
+import asyncio
+import aiofiles
+from typing import Dict
+from pathlib import Path
+
+_yaml_regex = re.compile(
+    r"^\s*" + r"(?:---|\+\+\+)" + r"(.*?)" + r"(?:---|\+\+\+)" + r"\s*(.+)$",
+    re.S | re.M,
+)
+
+def load_text(file_path, encoding='utf-8'):
+    with open(file_path, 'r', encoding=encoding) as file:
+        return file.read()
+
+async def load_text_async(file_path, encoding='utf-8'):
+    async with aiofiles.open(file_path, mode='r', encoding=encoding) as f:
+        content = await f.read()
+        return content
+
+def load_json(file_path, encoding='utf-8'):
+    return json.loads(load_text(file_path, encoding=encoding))
+
+async def load_json_async(file_path, encoding='utf-8'):
+    # async file open
+    content = await load_text_async(file_path, encoding=encoding)
+    return json.loads(content)
+
+def _find_global_config(prompty_path: Path = Path.cwd()) -> Path:
+    prompty_config = list(Path.cwd().glob("**/prompty.json"))
+
+    if len(prompty_config) > 0:
+        return sorted(
+            [
+                c
+                for c in prompty_config
+                if len(c.parent.parts) <= len(prompty_path.parts)
+            ],
+            key=lambda p: len(p.parts),
+        )[-1]
+    else:
+        return None
+
+
+def load_global_config(
+    prompty_path: Path = Path.cwd(), configuration: str = "default"
+) -> Dict[str, any]:
+    # prompty.config laying around?
+    config = _find_global_config(prompty_path)
+
+    # if there is one load it
+    if config is not None:
+        c = load_json(config)
+        if configuration in c:
+            return c[configuration]
+        else:
+            raise ValueError(f'Item "{configuration}" not found in "{config}"')
+
+    return {}
+
+
+async def load_global_config_async(
+    prompty_path: Path = Path.cwd(), configuration: str = "default"
+) -> Dict[str, any]:
+    # prompty.config laying around?
+    config = _find_global_config(prompty_path)
+
+    # if there is one load it
+    if config is not None:
+        c = await load_json_async(config)
+        if configuration in c:
+            return c[configuration]
+        else:
+            raise ValueError(f'Item "{configuration}" not found in "{config}"')
+
+    return {}
+
+
+def load_prompty(file_path, encoding='utf-8'):
+    contents = load_text(file_path, encoding=encoding)
+    return parse(contents)
+
+
+async def load_prompty_async(file_path, encoding="utf-8"):
+    contents = await load_text_async(file_path, encoding=encoding)
+    return parse(contents)
+
+
+def parse(contents):
+    global _yaml_regex
+
+    fmatter = ""
+    body = ""
+    result = _yaml_regex.search(contents)
+
+    if result:
+        fmatter = result.group(1)
+        body = result.group(2)
+    return {
+        "attributes": yaml.load(fmatter, Loader=yaml.FullLoader),
+        "body": body,
+        "frontmatter": fmatter,
+    }

prompty-0.1.33.dist-info/METADATA

@@ -0,0 +1,218 @@
+Metadata-Version: 2.1
+Name: prompty
+Version: 0.1.33
+Summary: Prompty is a new asset class and format for LLM prompts that aims to provide observability, understandability, and portability for developers. It includes spec, tooling, and a runtime. This Prompty runtime supports Python
+Author-Email: Seth Juarez <seth.juarez@microsoft.com>
+License: MIT
+Requires-Python: >=3.9
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: pydantic>=2.8.2
+Requires-Dist: jinja2>=3.1.4
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: click>=8.1.7
+Requires-Dist: aiofiles>=24.1.0
+Provides-Extra: azure
+Requires-Dist: azure-identity>=1.17.1; extra == "azure"
+Requires-Dist: openai>=1.35.10; extra == "azure"
+Provides-Extra: openai
+Requires-Dist: openai>=1.35.10; extra == "openai"
+Provides-Extra: serverless
+Requires-Dist: azure-ai-inference>=1.0.0b3; extra == "serverless"
+Description-Content-Type: text/markdown
+
+
+Prompty is an asset class and format for LLM prompts designed to enhance observability, understandability, and portability for developers. The primary goal is to accelerate the developer inner loop of prompt engineering and prompt source management in a cross-language and cross-platform implementation.
+
+The file format has a supporting toolchain with a VS Code extension and runtimes in multiple programming languages to simplify and accelerate your AI application development.
+
+The tooling comes together in three ways: the *prompty file asset*, the *VS Code extension tool*, and *runtimes* in multiple programming languages.
+
+## The Prompty File Format
+Prompty is a language agnostic prompt asset for creating prompts and engineering the responses. Learn more about the format [here](https://prompty.ai/docs/prompty-file-spec).
+
+Examples prompty file:
+```markdown
+---
+name: Basic Prompt
+description: A basic prompt that uses the GPT-3 chat API to answer questions
+authors:
+  - sethjuarez
+  - jietong
+model:
+  api: chat
+  configuration:
+    api_version: 2023-12-01-preview
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
+    azure_deployment: ${env:AZURE_OPENAI_DEPLOYMENT:gpt-35-turbo}
+sample:
+  firstName: Jane
+  lastName: Doe
+  question: What is the meaning of life?
+---
+system:
+You are an AI assistant who helps people find information.
+As the assistant, you answer questions briefly, succinctly, 
+and in a personable manner using markdown and even add some personal flair with appropriate emojis.
+
+# Customer
+You are helping {{firstName}} {{lastName}} to find answers to their questions.
+Use their name to address them in your responses.
+
+user:
+{{question}}
+```
+
+
+## The Prompty VS Code Extension
+Run Prompty files directly in VS Code. This Visual Studio Code extension offers an intuitive prompt playground within VS Code to streamline the prompt engineering process. You can find the Prompty extension in the Visual Studio Code Marketplace.
+
+Download the [VS Code extension here](https://marketplace.visualstudio.com/items?itemName=ms-toolsai.prompty).
+
+
+## Using this Prompty Runtime
+The Python runtime is a simple way to run your prompts in Python. The runtime is available as a Python package and can be installed using pip. Depending on the type of prompt you are running, you may need to install additional dependencies. The runtime is designed to be extensible and can be customized to fit your needs.
+
+```bash
+pip install prompty[azure]
+```
+
+Simple usage example:
+
+```python
+import prompty
+# import invoker
+import prompty.azure
+
+# execute the prompt
+response = prompty.execute("path/to/prompty/file")
+
+print(response)
+```
+
+## Available Invokers
+The Prompty runtime comes with a set of built-in invokers that can be used to execute prompts. These include:
+
+- `azure`: Invokes the Azure OpenAI API
+- `openai`: Invokes the OpenAI API
+- `serverless`: Invokes serverless models (like the ones on GitHub) using the [Azure AI Inference client library](https://learn.microsoft.com/en-us/python/api/overview/azure/ai-inference-readme?view=azure-python-preview) (currently only key based authentication is supported with more managed identity support coming soon)
+
+
+## Using Tracing in Prompty
+Prompty supports tracing to help you understand the execution of your prompts. This functionality is customizable and can be used to trace the execution of your prompts in a way that makes sense to you. Prompty has two default traces built in: `console_tracer` and `PromptyTracer`. The `console_tracer` writes the trace to the console, and the `PromptyTracer` writes the trace to a JSON file. You can also create your own tracer by creating your own hook.
+
+```python
+import prompty
+# import invoker
+import prompty.azure
+from prompty.tracer import trace, Tracer, console_tracer, PromptyTracer
+
+# add console tracer
+Tracer.add("console", console_tracer)
+
+# add PromptyTracer
+json_tracer = PromptyTracer(output_dir="path/to/output")
+Tracer.add("console", json_tracer.tracer)
+
+# execute the prompt
+response = prompty.execute("path/to/prompty/file")
+
+print(response)
+```
+
+You can also bring your own tracer by your own tracing hook. The `console_tracer` is the simplest example of a tracer. It writes the trace to the console.
+This is what it looks like:
+
+```python
+@contextlib.contextmanager
+def console_tracer(name: str) -> Iterator[Callable[[str, Any], None]]:
+    try:
+        print(f"Starting {name}")
+        yield lambda key, value: print(f"{key}:\n{json.dumps(value, indent=4)}")
+    finally:
+        print(f"Ending {name}")
+
+```
+
+It uses a context manager to define the start and end of the trace so you can do whatever setup and teardown you need. The `yield` statement returns a function that you can use to write the trace. The `console_tracer` writes the trace to the console using the `print` function.
+
+The `PromptyTracer` is a more complex example of a tracer. This tracer manages its internal state using a full class. Here's an example of the class based approach that writes each function trace to a JSON file:
+
+```python
+class SimplePromptyTracer:
+    def __init__(self, output_dir: str):
+        self.output_dir = output_dir
+        self.tracer = self._tracer
+
+    @contextlib.contextmanager
+    def tracer(self, name: str) -> Iterator[Callable[[str, Any], None]]:
+        trace = {}
+        try:
+            yield lambda key, value: trace.update({key: value})
+        finally:
+            with open(os.path.join(self.output_dir, f"{name}.json"), "w") as f:
+                json.dump(trace, f, indent=4)
+```
+
+The tracing mechanism is supported for all of the prompty runtime internals and can be used to trace the execution of the prompt along with all of the paramters. There is also a `@trace` decorator that can be used to trace the execution of any function external to the runtime. This is provided as a facility to trace the execution of the prompt and whatever supporting code you have.
+
+```python
+import prompty
+# import invoker
+import prompty.azure
+from prompty.tracer import trace, Tracer, PromptyTracer
+
+json_tracer = PromptyTracer(output_dir="path/to/output")
+Tracer.add("PromptyTracer", json_tracer.tracer)
+
+@trace
+def get_customer(customerId):
+    return {"id": customerId, "firstName": "Sally", "lastName": "Davis"}
+
+@trace
+def get_response(customerId, prompt):
+    customer = get_customer(customerId)
+
+    result = prompty.execute(
+        prompt,
+        inputs={"question": question, "customer": customer},
+    )
+    return {"question": question, "answer": result}
+
+```
+
+In this case, whenever this code is executed, a `.tracy` file will be created in the `path/to/output` directory. This file will contain the trace of the execution of the `get_response` function, the execution of the `get_customer` function, and the prompty internals that generated the response.
+
+## OpenTelemetry Tracing
+You can add OpenTelemetry tracing to your application using the same hook mechanism. In your application, you might create something like `trace_span` to trace the execution of your prompts:
+
+```python
+from opentelemetry import trace as oteltrace
+
+_tracer = "prompty"
+
+@contextlib.contextmanager
+def trace_span(name: str):
+    tracer = oteltrace.get_tracer(_tracer)
+    with tracer.start_as_current_span(name) as span:
+        yield lambda key, value: span.set_attribute(
+            key, json.dumps(value).replace("\n", "")
+        )
+
+# adding this hook to the prompty runtime
+Tracer.add("OpenTelemetry", trace_span)
+
+```
+
+This will produce spans during the execution of the prompt that can be sent to an OpenTelemetry collector for further analysis.
+
+## CLI
+The Prompty runtime also comes with a CLI tool that allows you to run prompts from the command line. The CLI tool is installed with the Python package.
+
+```bash
+prompty -s path/to/prompty/file -e .env
+```
+
+This will execute the prompt and print the response to the console. If there are any environment variables the CLI should take into account, you can pass those in via the `-e` flag. It also has default tracing enabled.
+
+## Contributing
+We welcome contributions to the Prompty project! This community led project is open to all contributors. The project can be found on [GitHub](https://github.com/Microsoft/prompty).

prompty-0.1.33.dist-info/RECORD

@@ -0,0 +1,22 @@
+prompty-0.1.33.dist-info/METADATA,sha256=nS7SXTCSlIm-LUsCFMAA9rvh6piDyFJLBkg_G96l85k,9105
+prompty-0.1.33.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+prompty-0.1.33.dist-info/entry_points.txt,sha256=a3i7Kvf--3DOkkv9VQpstwaNKgsnXwDGaPL18lPpKeI,60
+prompty-0.1.33.dist-info/licenses/LICENSE,sha256=KWSC4z9cfML_t0xThoQYjzTdcZQj86Y_mhXdatzU-KM,1052
+prompty/__init__.py,sha256=HCAvInBgNcIDO54rR4-RDIF4KUmGVQ2TRam_dS7xHEk,16561
+prompty/azure/__init__.py,sha256=WI8qeNWfxqggj21bznL-mxGUS-v67bUrunX0Lf2hsI8,295
+prompty/azure/executor.py,sha256=RJXMB0W7KcVvQ7l3xJaau7YM8PqOCQwuN4IwIe0sTLg,7930
+prompty/azure/processor.py,sha256=eWcHTLwxxBw7ZfK-rSf2cdljJgouxGXuRh_7EtV-MGk,4974
+prompty/cli.py,sha256=k8Rxm41fMFNvmnsX737UiN6v-7756tpoJPN4rPXMNcU,3726
+prompty/core.py,sha256=EvkXV_mH7Mj1skT21XMZ4VX-Jlwx6AF-WEJ9yPc50AE,13061
+prompty/invoker.py,sha256=O77E5iQ1552wQXxL8FhZGERbCi_0O3mDTd5Ozqw-O-E,8593
+prompty/openai/__init__.py,sha256=hbBhgCwB_uSq-1NWL02yiOiNkyi39-G-AyVlTSgKTkU,276
+prompty/openai/executor.py,sha256=qkFSMA-pWlA1c602Dx5aR1cFEOnYsUUp_E7P3zFhSPs,3644
+prompty/openai/processor.py,sha256=l9-91_CCgRtYvkwMO-jV6rkgeCA4gV_MFamQcvoNGQ0,2499
+prompty/parsers.py,sha256=zHqcRpFPUDG6BOI7ipaJf6yGc6ZbKnsLmO7jKEYNct4,5013
+prompty/renderers.py,sha256=80HNtCp3osgaLfhKxkG4j1kiRhJ727ITzT_yL5JLjEQ,1104
+prompty/serverless/__init__.py,sha256=xoXOTRXO8C631swNKaa-ek5_R3X-87bJpTm0z_Rsg6A,282
+prompty/serverless/executor.py,sha256=pwR_9itQUJ65Nk-sRJcfueJVZlvJdQ87R9FSHPCOi5o,5086
+prompty/serverless/processor.py,sha256=xjVvm4TSbp0PYbGzqLnm0PGxITlg-PIp1Uh4Rhvc1JY,2312
+prompty/tracer.py,sha256=pBCFs5xtWUOqF_QksHE3iRIIIxPyioxb4xHok-76ppQ,11169
+prompty/utils.py,sha256=jm7HEzOGk3zz8d5aquXK3zWIQWuDpBpJTzlz5sswtdg,2836
+prompty-0.1.33.dist-info/RECORD,,

{prompty-0.1.10.dist-info → prompty-0.1.33.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: pdm-backend (2.3.3)
+Generator: pdm-backend (2.4.3)
 Root-Is-Purelib: true
 Tag: py3-none-any

prompty-0.1.33.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+prompty = prompty.cli:run
+
+[gui_scripts]
+