opik 1.3.6__tar.gz

opik-1.3.6/PKG-INFO

@@ -0,0 +1,249 @@
+Metadata-Version: 2.2
+Name: opik
+Version: 1.3.6
+Summary: Comet tool for logging and evaluating LLM traces
+Home-page: https://www.comet.com
+Author: Comet ML Inc.
+Author-email: mail@comet.com
+License: Apache 2.0 License
+Project-URL: Source code, https://github.com/comet-ml/opik
+Keywords: opik
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: click
+Requires-Dist: httpx<0.28.0
+Requires-Dist: levenshtein<1.0.0
+Requires-Dist: litellm
+Requires-Dist: openai<2.0.0
+Requires-Dist: pydantic-settings<3.0.0,>=2.0.0
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: pytest
+Requires-Dist: rich
+Requires-Dist: sentry_sdk>=2.0.0
+Requires-Dist: tenacity
+Requires-Dist: tokenizers<0.21.0; python_version < "3.9.0"
+Requires-Dist: tqdm
+Requires-Dist: uuid6
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+<h1 align="center" style="border-bottom: none">
+    <div>
+        <a href="https://www.comet.com/site/products/opik/?from=llm&utm_source=opik&utm_medium=github&utm_content=header_img&utm_campaign=opik"><picture>
+            <source media="(prefers-color-scheme: dark)" srcset="/apps/opik-documentation/documentation/static/img/logo-dark-mode.svg">
+            <source media="(prefers-color-scheme: light)" srcset="/apps/opik-documentation/documentation/static/img/opik-logo.svg">
+            <img alt="Comet Opik logo" src="/apps/opik-documentation/documentation/static/img/opik-logo.svg" width="200" />
+        </picture></a>
+        <br>
+        Opik
+    </div>
+    Open source LLM evaluation framework<br>
+</h1>
+
+<p align="center">
+From RAG chatbots to code assistants to complex agentic pipelines and beyond, build LLM systems that run better, faster, and cheaper with tracing, evaluations, and dashboards.
+</p>
+
+<div align="center">
+
+[![Python SDK](https://img.shields.io/pypi/v/opik)](https://pypi.org/project/opik/)
+[![License](https://img.shields.io/github/license/comet-ml/opik)](https://github.com/comet-ml/opik/blob/main/LICENSE)
+[![Build](https://github.com/comet-ml/opik/actions/workflows/build_apps.yml/badge.svg)](https://github.com/comet-ml/opik/actions/workflows/build_apps.yml)
+<a target="_blank" href="https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/opik_quickstart.ipynb">
+
+  <!-- <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open Quickstart In Colab"/> -->
+</a>
+
+</div>
+
+<p align="center">
+    <a href="https://www.comet.com/site/products/opik/?from=llm&utm_source=opik&utm_medium=github&utm_content=website_button&utm_campaign=opik"><b>Website</b></a> •
+    <a href="https://chat.comet.com"><b>Slack community</b></a> •
+    <a href="https://x.com/Cometml"><b>Twitter</b></a> •
+    <a href="https://www.comet.com/docs/opik/?from=llm&utm_source=opik&utm_medium=github&utm_content=docs_button&utm_campaign=opik"><b>Documentation</b></a>
+</p>
+
+![Opik thumbnail](readme-thumbnail.png)
+
+## 🚀 What is Opik?
+
+Opik is an open-source platform for evaluating, testing and monitoring LLM applications. Built by [Comet](https://www.comet.com?from=llm&utm_source=opik&utm_medium=github&utm_content=what_is_opik_link&utm_campaign=opik).
+
+<br>
+
+You can use Opik for:
+* **Development:**
+
+  * **Tracing:** Track all LLM calls and traces during development and production ([Quickstart](https://www.comet.com/docs/opik/quickstart/?from=llm&utm_source=opik&utm_medium=github&utm_content=quickstart_link&utm_campaign=opik), [Integrations](https://www.comet.com/docs/opik/tracing/integrations/overview/?from=llm&utm_source=opik&utm_medium=github&utm_content=integrations_link&utm_campaign=opik)
+
+  * **Annotations:** Annotate your LLM calls by logging feedback scores using the [Python SDK](https://www.comet.com/docs/opik/tracing/annotate_traces/#annotating-traces-and-spans-using-the-sdk?from=llm&utm_source=opik&utm_medium=github&utm_content=sdk_link&utm_campaign=opik) or the [UI](https://www.comet.com/docs/opik/tracing/annotate_traces/#annotating-traces-through-the-ui?from=llm&utm_source=opik&utm_medium=github&utm_content=ui_link&utm_campaign=opik).
+
+  * **Playground:**: Try out different prompts and models in the [prompt playground](https://www.comet.com/docs/opik/evaluation/playground/?from=llm&utm_source=opik&utm_medium=github&utm_content=playground_link&utm_campaign=opik)
+
+* **Evaluation**: Automate the evaluation process of your LLM application:
+
+    * **Datasets and Experiments**: Store test cases and run experiments ([Datasets](https://www.comet.com/docs/opik/evaluation/manage_datasets/?from=llm&utm_source=opik&utm_medium=github&utm_content=datasets_link&utm_campaign=opik), [Evaluate your LLM Application](https://www.comet.com/docs/opik/evaluation/evaluate_your_llm/?from=llm&utm_source=opik&utm_medium=github&utm_content=eval_link&utm_campaign=opik))
+
+    * **LLM as a judge metrics**: Use Opik's LLM as a judge metric for complex issues like [hallucination detection](https://www.comet.com/docs/opik/evaluation/metrics/hallucination/?from=llm&utm_source=opik&utm_medium=github&utm_content=hallucination_link&utm_campaign=opik), [moderation](https://www.comet.com/docs/opik/evaluation/metrics/moderation/?from=llm&utm_source=opik&utm_medium=github&utm_content=moderation_link&utm_campaign=opik) and RAG evaluation ([Answer Relevance](https://www.comet.com/docs/opik/evaluation/metrics/answer_relevance/?from=llm&utm_source=opik&utm_medium=github&utm_content=alex_link&utm_campaign=opik), [Context Precision](https://www.comet.com/docs/opik/evaluation/metrics/context_precision/?from=llm&utm_source=opik&utm_medium=github&utm_content=context_link&utm_campaign=opik)
+
+    * **CI/CD integration**: Run evaluations as part of your CI/CD pipeline using our [PyTest integration](https://www.comet.com/docs/opik/testing/pytest_integration/?from=llm&utm_source=opik&utm_medium=github&utm_content=pytest_link&utm_campaign=opik)
+
+* **Production Monitoring**:
+    
+    * **Log all your production traces**: Opik has been designed to support high volumes of traces, making it easy to monitor your production applications.
+    
+    * **Monitoring dashboards**: Review your feedback scores, trace count and tokens over time in the [Opik Dashboard](https://www.comet.com/docs/opik/self-host/opik_dashboard/?from=llm&utm_source=opik&utm_medium=github&utm_content=dashboard_link&utm_campaign=opik).
+
+> [!TIP]  
+> If you are looking for features that Opik doesn't have today, please raise a new [Feature request](https://github.com/comet-ml/opik/issues/new/choose) 🚀
+
+<br>
+
+## 🛠️ Installation
+Opik is available as a fully open source local installation or using Comet.com as a hosted solution.
+The easiest way to get started with Opik is by creating a free Comet account at [comet.com](https://www.comet.com/signup?from=llm&utm_source=opik&utm_medium=github&utm_content=install&utm_campaign=opik).
+
+If you'd like to self-host Opik, you can do so by cloning the repository and starting the platform using Docker Compose:
+
+```bash
+# Clone the Opik repository
+git clone https://github.com/comet-ml/opik.git
+
+# Navigate to the opik/deployment/docker-compose directory
+cd opik/deployment/docker-compose
+
+# Start the Opik platform
+docker compose up --detach
+
+# You can now visit http://localhost:5173 on your browser!
+```
+
+For more information about the different deployment options, please see our deployment guides:
+
+| Installation methods | Docs link |
+| ------------------- | --------- |
+| Local instance | [![Local Deployment](https://img.shields.io/badge/Local%20Deployments-%232496ED?style=flat&logo=docker&logoColor=white)](https://www.comet.com/docs/opik/self-host/local_deployment?from=llm&utm_source=opik&utm_medium=github&utm_content=self_host_link&utm_campaign=opik)
+| Kubernetes | [![Kubernetes](https://img.shields.io/badge/Kubernetes-%23326ce5.svg?&logo=kubernetes&logoColor=white)](https://www.comet.com/docs/opik/self-host/kubernetes/#kubernetes-installation?from=llm&utm_source=opik&utm_medium=github&utm_content=kubernetes_link&utm_campaign=opik)
+
+
+## 🏁 Get Started
+
+To get started, you will need to first install the Python SDK:
+
+```bash
+pip install opik
+```
+
+Once the SDK is installed, you can configure it by running the `opik configure` command:
+
+```bash
+opik configure
+```
+
+This will allow you to configure Opik locally by setting the correct local server address or if you're using the Cloud platform by setting the API Key
+
+> [!TIP]  
+> You can also call the `opik.configure(use_local=True)` method from your Python code to configure the SDK to run on the local installation.
+
+You are now ready to start logging traces using the [Python SDK](https://www.comet.com/docs/opik/python-sdk-reference/?from=llm&utm_source=opik&utm_medium=github&utm_content=sdk_link2&utm_campaign=opik).
+
+### 📝 Logging Traces
+
+The easiest way to get started is to use one of our integrations. Opik supports:
+
+| Integration | Description                                                                  | Documentation                                                                                                                                                      | Try in Colab                                                                                                                                                                                                                      |
+|-------------|------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| OpenAI      | Log traces for all OpenAI LLM calls                                          | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/openai/?utm_source=opik&utm_medium=github&utm_content=openai_link&utm_campaign=opik)          | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/openai.ipynb)      |
+| LiteLLM     | Call any LLM model using the OpenAI format                                   | [Documentation](/tracing/integrations/litellm.md)                                                                                                                  | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/litellm.ipynb)     |
+| LangChain   | Log traces for all LangChain LLM calls                                       | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/langchain/?utm_source=opik&utm_medium=github&utm_content=langchain_link&utm_campaign=opik)    | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/langchain.ipynb)   |
+| Haystack    | Log traces for all Haystack calls                                            | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/haystack/?utm_source=opik&utm_medium=github&utm_content=haystack_link&utm_campaign=opik)      | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/haystack.ipynb)    |
+| Anthropic   | Log traces for all Anthropic LLM calls                                       | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/anthropic?utm_source=opik&utm_medium=github&utm_content=anthropic_link&utm_campaign=opik)     | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/anthropic.ipynb)   |
+| Bedrock     | Log traces for all Bedrock LLM calls                                         | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/bedrock?utm_source=opik&utm_medium=github&utm_content=bedrock_link&utm_campaign=opik)         | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/bedrock.ipynb)     |
+| CrewAI      | Log traces for all CrewAI calls                                              | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/crewai?utm_source=opik&utm_medium=github&utm_content=crewai_link&utm_campaign=opik)           | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/crewai.ipynb)      |
+| DSPy        | Log traces for all DSPy runs                                                 | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/dspy?utm_source=opik&utm_medium=github&utm_content=dspy_link&utm_campaign=opik)               | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/dspy.ipynb)        |
+| Gemini      | Log traces for all Gemini LLM calls                                          | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/gemini?utm_source=opik&utm_medium=github&utm_content=gemini_link&utm_campaign=opik)           | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/gemini.ipynb)      |
+| Groq        | Log traces for all Groq LLM calls                                            | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/groq?utm_source=opik&utm_medium=github&utm_content=groq_link&utm_campaign=opik)               | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/groq.ipynb)        |
+| LangGraph   | Log traces for all LangGraph executions                                      | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/langgraph/?utm_source=opik&utm_medium=github&utm_content=langchain_link&utm_campaign=opik)    | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/langgraph.ipynb)   |
+| LlamaIndex  | Log traces for all LlamaIndex LLM calls                                      | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/llama_index?utm_source=opik&utm_medium=github&utm_content=llama_index_link&utm_campaign=opik) | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/llama-index.ipynb) |
+| Ollama      | Log traces for all Ollama LLM calls                                          | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/ollama?utm_source=opik&utm_medium=github&utm_content=ollama_link&utm_campaign=opik)           | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/ollama.ipynb)      |
+| Predibase   | Fine-tune and serve open-source Large Language Models                        | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/predibase?utm_source=opik&utm_medium=github&utm_content=predibase_link&utm_campaign=opik)     | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/predibase.ipynb)   |
+| Ragas       | Evaluation framework for your Retrieval Augmented Generation (RAG) pipelines | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/ragas?utm_source=opik&utm_medium=github&utm_content=ragas_link&utm_campaign=opik)             | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/ragas.ipynb)       |
+| watsonx     | Log traces for all watsonx LLM calls                                         | [Documentation](https://www.comet.com/docs/opik/tracing/integrations/watsonx?utm_source=opik&utm_medium=github&utm_content=watsonx_link&utm_campaign=opik)         | [![Open Quickstart In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/opik/blob/master/apps/opik-documentation/documentation/docs/cookbook/watsonx.ipynb)     |
+
+> [!TIP]  
+> If the framework you are using is not listed above, feel free to [open an issue](https://github.com/comet-ml/opik/issues) or submit a PR with the integration.
+
+If you are not using any of the frameworks above, you can also use the `track` function decorator to [log traces](https://www.comet.com/docs/opik/tracing/log_traces/?from=llm&utm_source=opik&utm_medium=github&utm_content=traces_link&utm_campaign=opik):
+
+```python
+import opik
+
+opik.configure(use_local=True) # Run locally
+
+@opik.track
+def my_llm_function(user_question: str) -> str:
+    # Your LLM code here
+
+    return "Hello"
+```
+
+> [!TIP]  
+> The track decorator can be used in conjunction with any of our integrations and can also be used to track nested function calls.
+
+### 🧑‍⚖️ LLM as a Judge metrics
+
+The Python Opik SDK includes a number of LLM as a judge metrics to help you evaluate your LLM application. Learn more about it in the [metrics documentation](https://www.comet.com/docs/opik/evaluation/metrics/overview/?from=llm&utm_source=opik&utm_medium=github&utm_content=metrics_2_link&utm_campaign=opik).
+
+To use them, simply import the relevant metric and use the `score` function:
+
+```python
+from opik.evaluation.metrics import Hallucination
+
+metric = Hallucination()
+score = metric.score(
+    input="What is the capital of France?",
+    output="Paris",
+    context=["France is a country in Europe."]
+)
+print(score)
+```
+
+Opik also includes a number of pre-built heuristic metrics as well as the ability to create your own. Learn more about it in the [metrics documentation](https://www.comet.com/docs/opik/evaluation/metrics/overview?from=llm&utm_source=opik&utm_medium=github&utm_content=metrics_3_link&utm_campaign=opik).
+
+### 🔍 Evaluating your LLM Application
+
+Opik allows you to evaluate your LLM application during development through [Datasets](https://www.comet.com/docs/opik/evaluation/manage_datasets/?from=llm&utm_source=opik&utm_medium=github&utm_content=datasets_2_link&utm_campaign=opik) and [Experiments](https://www.comet.com/docs/opik/evaluation/evaluate_your_llm/?from=llm&utm_source=opik&utm_medium=github&utm_content=experiments_link&utm_campaign=opik).
+
+You can also run evaluations as part of your CI/CD pipeline using our [PyTest integration](https://www.comet.com/docs/opik/testing/pytest_integration/?from=llm&utm_source=opik&utm_medium=github&utm_content=pytest_2_link&utm_campaign=opik).
+
+## 🤝 Contributing
+
+There are many ways to contribute to Opik:
+
+* Submit [bug reports](https://github.com/comet-ml/opik/issues) and [feature requests](https://github.com/comet-ml/opik/issues)
+* Review the documentation and submit [Pull Requests](https://github.com/comet-ml/opik/pulls) to improve it
+* Speaking or writing about Opik and [letting us know](https://chat.comet.com)
+* Upvoting [popular feature requests](https://github.com/comet-ml/opik/issues?q=is%3Aissue+is%3Aopen+label%3A%22enhancement%22) to show your support
+
+To learn more about how to contribute to Opik, please see our [contributing guidelines](CONTRIBUTING.md).

opik-1.3.6/README.md

@@ -0,0 +1,3 @@
+# Opik Python SDK
+
+If you would like to contribute to the Opik python SDK, please refer to the [Contribution guide](./CONTRIBUTING.md).

opik-1.3.6/pyproject.toml

@@ -0,0 +1,6 @@
+[tool.mypy]
+follow_imports = "skip"
+ignore_missing_imports = true
+disallow_untyped_defs = true
+disallow_untyped_calls = true
+check_untyped_defs = true

opik-1.3.6/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

opik-1.3.6/setup.py

@@ -0,0 +1,72 @@
+import os
+
+from setuptools import find_packages, setup
+
+project_urls = {"Source code": "https://github.com/comet-ml/opik"}
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+version = os.environ.get("VERSION")
+if version is None:
+    version_file = os.path.join(HERE, "..", "..", "version.txt")
+    if os.path.exists(version_file):
+        with open(version_file) as fp:
+            version = fp.read().strip()
+    else:
+        version = "0.0.1"
+
+setup(
+    author="Comet ML Inc.",
+    author_email="mail@comet.com",
+    python_requires=">=3.8",
+    classifiers=[
+        "Development Status :: 2 - Pre-Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Apache Software License",
+        "Natural Language :: English",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+    ],
+    description="Comet tool for logging and evaluating LLM traces",
+    long_description=open(
+        os.path.join(HERE, "..", "..", "README.md"), encoding="utf-8"
+    ).read(),
+    long_description_content_type="text/markdown",
+    install_requires=[
+        "click",
+        "httpx<0.28.0",
+        "levenshtein<1.0.0",
+        "litellm",
+        "openai<2.0.0",
+        "pydantic-settings>=2.0.0,<3.0.0",
+        "pydantic>=2.0.0,<3.0.0",
+        "pytest",
+        "rich",
+        "sentry_sdk>=2.0.0",
+        "tenacity",
+        "tokenizers<0.21.0 ; python_version<'3.9.0'",  # no 3.8 support starting from 0.21.0
+        "tqdm",
+        "uuid6",
+    ],
+    entry_points={
+        "pytest11": [
+            "opik = opik.plugins.pytest.hooks",
+        ],
+        "console_scripts": ["opik = opik.cli:cli"],
+    },
+    keywords="opik",
+    name="opik",
+    include_package_data=True,
+    package_data={"opik": ["py.typed"]},
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    url="https://www.comet.com",
+    project_urls=project_urls,
+    version=version,
+    zip_safe=False,
+    license="Apache 2.0 License",
+)

opik-1.3.6/src/opik/__init__.py

@@ -0,0 +1,45 @@
+from . import _logging, error_tracking, package_version
+from .api_objects.dataset import Dataset
+from .api_objects.experiment.experiment_item import (
+    ExperimentItemContent,
+    ExperimentItemReferences,
+)
+from .api_objects.opik_client import Opik
+from .api_objects.prompt import Prompt
+from .api_objects.span import Span
+from .api_objects.trace import Trace
+from .configurator.configure import configure
+from .decorator.tracker import flush_tracker, track
+from .evaluation import evaluate, evaluate_experiment, evaluate_prompt
+from .integrations.sagemaker import auth as sagemaker_auth
+from .plugins.pytest.decorator import llm_unit
+
+_logging.setup()
+
+__version__ = package_version.VERSION
+__all__ = [
+    "__version__",
+    "evaluate",
+    "evaluate_prompt",
+    "evaluate_experiment",
+    "ExperimentItemContent",
+    "ExperimentItemReferences",
+    "track",
+    "flush_tracker",
+    "Opik",
+    "Trace",
+    "Span",
+    "Dataset",
+    "llm_unit",
+    "configure",
+    "Prompt",
+]
+
+sagemaker_auth.setup_aws_sagemaker_session_hook()
+
+
+if (
+    error_tracking.enabled_in_config()
+    and error_tracking.randomized_should_enable_reporting()
+):
+    error_tracking.setup_sentry_error_tracker()

opik-1.3.6/src/opik/_logging.py

@@ -0,0 +1,81 @@
+from typing import Callable, Any, Optional
+import functools
+import logging
+
+
+from . import config
+
+CONSOLE_MSG_FORMAT = "OPIK: %(message)s"
+DEBUG_MSG_FORMAT = "%(asctime)s [%(process)d-%(processName)s:%(thread)d] %(relativeCreated)d OPIK %(levelname)s [%(filename)s:%(lineno)d]: %(message)s"
+# 1MB, to prevent logger from frequent writing hundreds of megabytes in DEBUG mode
+# when batches are big and payloads are heavy (e.g. base64 encoded data)
+MAX_MESSAGE_LENGTH = 1024 * 1024
+
+
+class TruncateFormatter(logging.Formatter):
+    def __init__(
+        self,
+        fmt: str,
+        datefmt: Optional[str] = None,
+        max_length: int = MAX_MESSAGE_LENGTH,
+    ) -> None:
+        super().__init__(fmt, datefmt)
+        self.max_length = max_length
+
+    def format(self, record: logging.LogRecord) -> str:
+        result = super().format(record)
+
+        if len(result) > self.max_length:
+            result = result[: self.max_length] + "... (truncated)."
+
+        return result
+
+
+def setup() -> None:
+    opik_root_logger = logging.getLogger("opik")
+    opik_root_logger.propagate = False
+
+    config_ = config.OpikConfig()
+
+    console_handler = logging.StreamHandler()
+    console_level = config_.console_logging_level
+    console_handler.setLevel(console_level)
+    message_format = (
+        DEBUG_MSG_FORMAT if console_level == "DEBUG" else CONSOLE_MSG_FORMAT
+    )
+    console_handler.setFormatter(TruncateFormatter(message_format))
+    opik_root_logger.addHandler(console_handler)
+
+    root_level = console_handler.level
+
+    if config_.file_logging_level is not None:
+        file_handler = logging.FileHandler(config_.logging_file)
+        file_level = config_.file_logging_level
+        file_handler.setLevel(file_level)
+        file_handler.setFormatter(TruncateFormatter(DEBUG_MSG_FORMAT))
+        opik_root_logger.addHandler(file_handler)
+
+        root_level = min(root_level, file_handler.level)
+
+    opik_root_logger.setLevel(level=root_level)
+
+
+def convert_exception_to_log_message(
+    message: str,
+    logger: logging.Logger,
+    return_on_exception: Any = None,
+    logging_level: int = logging.ERROR,
+    **log_kwargs: Any,
+) -> Callable:
+    def decorator(function: Callable) -> Any:
+        @functools.wraps(function)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            try:
+                return function(*args, **kwargs)
+            except Exception:
+                logger.log(logging_level, message, **log_kwargs)
+                return return_on_exception
+
+        return wrapper
+
+    return decorator

opik-1.3.6/src/opik/api_key/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# *******************************************************
+#   ____                     _               _
+#  / ___|___  _ __ ___   ___| |_   _ __ ___ | |
+# | |   / _ \| '_ ` _ \ / _ \ __| | '_ ` _ \| |
+# | |__| (_) | | | | | |  __/ |_ _| | | | | | |
+#  \____\___/|_| |_| |_|\___|\__(_)_| |_| |_|_|
+#
+#  Sign up for free at https://www.comet.com
+#  Copyright (C) 2015-2024 Comet ML INC
+#  This file can not be copied and/or distributed
+#  without the express permission of Comet ML Inc.
+# *******************************************************

opik-1.3.6/src/opik/api_key/base64_helper.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# *******************************************************
+#   ____                     _               _
+#  / ___|___  _ __ ___   ___| |_   _ __ ___ | |
+# | |   / _ \| '_ ` _ \ / _ \ __| | '_ ` _ \| |
+# | |__| (_) | | | | | |  __/ |_ _| | | | | | |
+#  \____\___/|_| |_| |_|\___|\__(_)_| |_| |_|_|
+#
+#  Sign up for free at https://www.comet.com
+#  Copyright (C) 2015-2024 Comet ML INC
+#  This file can not be copied and/or distributed
+#  without the express permission of Comet ML Inc.
+# *******************************************************
+import base64
+
+
+def decode_base64(data: str, fix_padding: bool = True) -> bytes:
+    if fix_padding:
+        missing_padding = len(data) % 4
+        if missing_padding and data.endswith("="):
+            # wrong padding
+            data = data.replace("=", "")
+            return decode_base64(data)
+
+        if missing_padding:
+            data += "=" * (4 - missing_padding)
+    return base64.b64decode(data, validate=True)

opik-1.3.6/src/opik/api_key/opik_api_key.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# *******************************************************
+#   ____                     _               _
+#  / ___|___  _ __ ___   ___| |_   _ __ ___ | |
+# | |   / _ \| '_ ` _ \ / _ \ __| | '_ ` _ \| |
+# | |__| (_) | | | | | |  __/ |_ _| | | | | | |
+#  \____\___/|_| |_| |_|\___|\__(_)_| |_| |_|_|
+#
+#  Sign up for free at https://www.comet.com
+#  Copyright (C) 2015-2024 Comet ML INC
+#  This file can not be copied and/or distributed
+#  without the express permission of Comet ML Inc.
+# *******************************************************
+import json
+import logging
+from typing import Any, Dict, Optional
+
+from ..logging_messages import (
+    PARSE_API_KEY_EMPTY_EXPECTED_ATTRIBUTES,
+    PARSE_API_KEY_EMPTY_KEY,
+    PARSE_API_KEY_TOO_MANY_PARTS,
+)
+from .base64_helper import decode_base64
+
+LOGGER = logging.getLogger(__name__)
+
+DELIMITER_CHAR = "*"
+
+
+class OpikApiKey:
+    """
+    This is Opik API key parser module which is able to parse enhanced API key format. The format as following:
+    initial 25 chars apiKey + DELIMITER_CHAR + base64 encoded OPIK_BASE_URL and other attributes as JSON dictionary.
+
+    The logic of this module is shared among comet_ml, comet_mpm, and opik projects.
+    Please do not change this module without synchronization with mentioned projects.
+    """
+
+    def __init__(
+        self,
+        api_key_raw: str,
+        api_key: Optional[str] = None,
+        attributes: Optional[Dict[str, Any]] = None,
+    ):
+        self._api_key_raw = api_key_raw
+        self._api_key = api_key
+        self._attributes = attributes
+
+    @property
+    def api_key(self) -> Optional[str]:
+        return self._api_key_raw
+
+    @property
+    def short_api_key(self) -> Optional[str]:
+        if self._api_key is not None:
+            return self._api_key
+        return self._api_key_raw
+
+    @property
+    def base_url(self) -> Optional[str]:
+        if self["baseUrl"] is not None:
+            return str(self["baseUrl"])
+        else:
+            return None
+
+    def __getitem__(self, key: str) -> Any:
+        if self._attributes is not None:
+            return self._attributes.get(key, None)
+
+        return None
+
+
+def parse_api_key(raw_key: str) -> Optional[OpikApiKey]:
+    if raw_key is None or len(raw_key) == 0:
+        LOGGER.debug(PARSE_API_KEY_EMPTY_KEY)
+        return None
+
+    parts = raw_key.split(DELIMITER_CHAR)
+    size = len(parts)
+    if size == 1:
+        LOGGER.debug("Opik API key doesn't have attributes associated")
+        return OpikApiKey(api_key_raw=raw_key)
+    elif size == 2:
+        attr_string = parts[1]
+        if len(attr_string) > 0:
+            data = decode_base64(attr_string)
+            attributes = json.loads(data)
+        else:
+            # edge case - delimiter found but no encoded JSON afterward
+            LOGGER.warning(PARSE_API_KEY_EMPTY_EXPECTED_ATTRIBUTES % raw_key)
+            raw_key = parts[0]  # remove obsolete delimiter
+            attributes = None
+
+        return OpikApiKey(api_key_raw=raw_key, api_key=parts[0], attributes=attributes)
+
+    LOGGER.warning(PARSE_API_KEY_TOO_MANY_PARTS, size, raw_key)
+    return None

opik-1.3.6/src/opik/api_objects/constants.py

@@ -0,0 +1,6 @@
+FEEDBACK_SCORE_SOURCE_SDK = "sdk"
+DATASET_SOURCE_SDK = "sdk"
+
+FEEDBACK_SCORES_MAX_BATCH_SIZE = 1000
+EXPERIMENT_ITEMS_MAX_BATCH_SIZE = 1000
+DATASET_ITEMS_MAX_BATCH_SIZE = 1000

opik-1.3.6/src/opik/api_objects/dataset/__init__.py

@@ -0,0 +1,4 @@
+from .dataset import Dataset
+
+
+__all__ = ["Dataset"]