opik 0.1.3__tar.gz

opik-0.1.3/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.1
+Name: opik
+Version: 0.1.3
+Summary: Comet tool for logging and evaluating LLM traces
+Home-page: https://www.comet.com
+Author: Comet ML Inc.
+Author-email: mail@comet.com
+License: MIT
+Project-URL: Source code, https://github.com/comet-ml/opik
+Keywords: opik
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: httpx<1.0.0
+Requires-Dist: langchain_community<1.0.0
+Requires-Dist: langchain_openai<1.0.0
+Requires-Dist: levenshtein~=0.25.1
+Requires-Dist: openai<2.0.0
+Requires-Dist: pandas<3.0.0,>=2.0.0
+Requires-Dist: pydantic-settings<3.0.0,>=2.0.0
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: pytest
+Requires-Dist: tqdm
+Requires-Dist: uuid7<1.0.0
+Requires-Dist: rich
+
+# opik-python
+
+To install package in development mode run `pip install -e .` from the repository root directory.
+
+# Linters
+
+Before pushing your changes please run `pre-commit run --all-files` from the python/sdk directory.
+It will install and run required linters.

opik-0.1.3/README.md

@@ -0,0 +1,8 @@
+# opik-python
+
+To install package in development mode run `pip install -e .` from the repository root directory.
+
+# Linters
+
+Before pushing your changes please run `pre-commit run --all-files` from the python/sdk directory.
+It will install and run required linters.

opik-0.1.3/pyproject.toml

@@ -0,0 +1,6 @@
+[tool.mypy]
+follow_imports = "skip"
+ignore_missing_imports = true
+disallow_untyped_defs = true
+disallow_untyped_calls = true
+check_untyped_defs = true

opik-0.1.3/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

opik-0.1.3/setup.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+
+from setuptools import find_packages, setup
+import os
+
+project_urls = {"Source code": "https://github.com/comet-ml/opik"}
+this_directory = Path(__file__).parent
+
+setup(
+    author="Comet ML Inc.",
+    author_email="mail@comet.com",
+    python_requires=">=3.8",
+    classifiers=[
+        "Development Status :: 2 - Pre-Alpha",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: MIT License",
+        "Natural Language :: English",
+        "Programming Language :: Python :: 3 :: Only",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+    ],
+    description="Comet tool for logging and evaluating LLM traces",
+    long_description=open("README.md", encoding="utf-8").read(),
+    long_description_content_type="text/markdown",
+    install_requires=[
+        "httpx<1.0.0",
+        "langchain_community<1.0.0",
+        "langchain_openai<1.0.0",
+        "levenshtein~=0.25.1",
+        "openai<2.0.0",
+        "pandas>=2.0.0,<3.0.0",
+        "pydantic-settings>=2.0.0,<3.0.0",
+        "pydantic>=2.0.0,<3.0.0",
+        "pytest",
+        "tqdm",
+        "uuid7<1.0.0",
+        "rich",
+    ],
+    entry_points={
+        "pytest11": [
+            "opik = opik.plugins.pytest.hooks",
+        ]
+    },
+    include_package_data=True,
+    keywords="opik",
+    name="opik",
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    url="https://www.comet.com",
+    project_urls=project_urls,
+    version=os.environ.get("VERSION", "0.0.1"),
+    zip_safe=False,
+    license="MIT",
+)

opik-0.1.3/src/opik/__init__.py

@@ -0,0 +1,24 @@
+from .decorator.tracker import track, flush_tracker
+from .api_objects.opik_client import Opik
+from .api_objects.trace import Trace
+from .api_objects.span import Span
+from .api_objects.dataset.dataset_item import DatasetItem
+from .api_objects.dataset import Dataset
+from . import _logging
+from . import package_version
+from .plugins.pytest.decorator import llm_unit
+
+_logging.setup()
+
+__version__ = package_version.VERSION
+__all__ = [
+    "__version__",
+    "track",
+    "flush_tracker",
+    "Opik",
+    "Trace",
+    "Span",
+    "DatasetItem",
+    "Dataset",
+    "llm_unit",
+]

opik-0.1.3/src/opik/_logging.py

@@ -0,0 +1,34 @@
+import logging
+
+import sys
+
+from . import config
+
+CONSOLE_MSG_FORMAT = "OPIK: %(message)s"
+
+FILE_MSG_FORMAT = "%(asctime)s OPIK %(levelname)s: %(message)s"
+
+
+def setup() -> None:
+    opik_root_logger = logging.getLogger("opik")
+    config_ = config.OpikConfig()
+
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_level = config_.console_logging_level
+    console_handler.setLevel(console_level)
+    console_handler.setFormatter(logging.Formatter(CONSOLE_MSG_FORMAT))
+
+    opik_root_logger.addHandler(console_handler)
+
+    root_level = console_handler.level
+
+    if config_.file_logging_level is not None:
+        file_handler = logging.FileHandler(config_.logging_file)
+        file_level = config_.file_logging_level
+        file_handler.setLevel(file_level)
+        file_handler.setFormatter(logging.Formatter(FILE_MSG_FORMAT))
+        opik_root_logger.addHandler(file_handler)
+
+        root_level = min(root_level, file_handler.level)
+
+    opik_root_logger.setLevel(level=root_level)

opik-0.1.3/src/opik/api_objects/constants.py

@@ -0,0 +1,6 @@
+FEEDBACK_SCORE_SOURCE_SDK = "sdk"
+DATASET_SOURCE_SDK = "sdk"
+
+FEEDBACK_SCORES_MAX_BATCH_SIZE = 1000
+EXPERIMENT_ITEMS_MAX_BATCH_SIZE = 1000
+DATASET_ITEMS_MAX_BATCH_SIZE = 1000

opik-0.1.3/src/opik/api_objects/dataset/__init__.py

@@ -0,0 +1,4 @@
+from .dataset import Dataset
+
+
+__all__ = ["Dataset"]

opik-0.1.3/src/opik/api_objects/dataset/converters.py

@@ -0,0 +1,71 @@
+import pandas as pd
+import json
+
+from typing import List, Callable, Any, Dict
+
+from . import dataset_item
+
+ItemConstructor = Callable[[Any], dataset_item.DatasetItem]
+
+
+def to_pandas(
+    items: List[dataset_item.DatasetItem], keys_mapping: Dict[str, str]
+) -> pd.DataFrame:
+    new_item_dicts = []
+
+    for item in items:
+        new_item_dict = {
+            keys_mapping.get(key, key): value for key, value in item.__dict__.items()
+        }
+        new_item_dicts.append(new_item_dict)
+
+    return pd.DataFrame(new_item_dicts)
+
+
+def from_pandas(
+    dataframe: pd.DataFrame,
+    keys_mapping: Dict[str, str],
+    ignore_keys: List[str],
+) -> List[dataset_item.DatasetItem]:
+    result = []
+    ignore_keys = [] if ignore_keys is None else ignore_keys
+    for _, row in dataframe.iterrows():
+        item_kwargs = {
+            keys_mapping.get(key, key): value
+            for key, value in row.items()
+            if key not in ignore_keys
+        }
+        result.append(dataset_item.DatasetItem(**item_kwargs))
+
+    return result
+
+
+def to_json(items: List[dataset_item.DatasetItem], keys_mapping: Dict[str, str]) -> str:
+    new_item_dicts = []
+
+    for item in items:
+        item_dict = item.__dict__
+        new_item_dict = {
+            keys_mapping.get(key, key): value for key, value in item_dict.items()
+        }
+        new_item_dicts.append(new_item_dict)
+
+    result: str = json.dumps(new_item_dicts, indent=2)
+    return result
+
+
+def from_json(
+    value: str, keys_mapping: Dict[str, str], ignore_keys: List[str]
+) -> List[dataset_item.DatasetItem]:
+    result = []
+    item_dicts: List[Dict[str, Any]] = json.loads(value)
+
+    for item_dict in item_dicts:
+        item_kwargs = {
+            keys_mapping.get(key, key): value
+            for key, value in item_dict.items()
+            if key not in ignore_keys
+        }
+        result.append(dataset_item.DatasetItem(**item_kwargs))
+
+    return result

opik-0.1.3/src/opik/api_objects/dataset/dataset.py

@@ -0,0 +1,227 @@
+import logging
+import json
+from typing import Optional, Any, List, Dict, Union, Sequence
+
+from opik.rest_api import client as rest_api_client
+from opik.rest_api.types import dataset_item as rest_dataset_item
+from opik import exceptions
+
+from .. import helpers, constants
+from . import dataset_item, converters
+
+import pandas
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Dataset:
+    def __init__(
+        self,
+        name: str,
+        description: Optional[str],
+        rest_client: rest_api_client.OpikApi,
+    ) -> None:
+        """
+        A Dataset object. This object should not be created directly, instead use :meth:`opik.Opik.create_dataset` or :meth:`opik.Opik.get_dataset`.
+        """
+        self._name = name
+        self._description = description
+        self._rest_client = rest_client
+
+    @property
+    def name(self) -> str:
+        """The name of the dataset."""
+        return self._name
+
+    @property
+    def description(self) -> Optional[str]:
+        """The description of the dataset."""
+        return self._description
+
+    def insert(
+        self, items: Sequence[Union[dataset_item.DatasetItem, Dict[str, Any]]]
+    ) -> None:
+        """
+        Insert new items into the dataset.
+
+        Args:
+            items: List of DatasetItem objects or dicts (which will be converted to DatasetItem objects)
+                to add to the dataset.
+        """
+        items: List[dataset_item.DatasetItem] = [  # type: ignore
+            (dataset_item.DatasetItem(**item) if isinstance(item, dict) else item)
+            for item in items
+        ]
+
+        rest_items = [
+            rest_dataset_item.DatasetItem(
+                id=item.id if item.id is not None else helpers.generate_id(),  # type: ignore
+                input=item.input,  # type: ignore
+                expected_output=item.expected_output,  # type: ignore
+                metadata=item.metadata,  # type: ignore
+                trace_id=item.trace_id,  # type: ignore
+                span_id=item.span_id,  # type: ignore
+                source=item.source,  # type: ignore
+            )
+            for item in items
+        ]
+
+        batches = helpers.list_to_batches(
+            rest_items, batch_size=constants.DATASET_ITEMS_MAX_BATCH_SIZE
+        )
+
+        for batch in batches:
+            LOGGER.debug("Sending dataset items batch: %s", batch)
+            self._rest_client.datasets.create_or_update_dataset_items(
+                dataset_name=self._name, items=batch
+            )
+
+    def update(self, items: List[dataset_item.DatasetItem]) -> None:
+        """
+        Update existing items in the dataset.
+
+        Args:
+            items: List of DatasetItem objects to update in the dataset. You need to provide the full item object as it will override what has been supplied previously.
+
+        Raises:
+            DatasetItemUpdateOperationRequiresItemId: If any item in the list is missing an id.
+        """
+        for item in items:
+            if item.id is None:  # TODO: implement proper exception
+                raise exceptions.DatasetItemUpdateOperationRequiresItemId(
+                    "Missing id for dataset item to update: %s", item
+                )
+
+        self.insert(items)
+
+    def delete(self, items_ids: List[str]) -> None:
+        """
+        Delete items from the dataset.
+
+        Args:
+            items_ids: List of item ids to delete.
+        """
+        batches = helpers.list_to_batches(
+            items_ids, batch_size=constants.DATASET_ITEMS_MAX_BATCH_SIZE
+        )
+
+        for batch in batches:
+            LOGGER.debug("Deleting dataset items batch: %s", batch)
+            self._rest_client.datasets.delete_dataset_items(item_ids=batch)
+
+    def clear(self) -> None:
+        """
+        Delete all items from the given dataset.
+        """
+        all_items = self.get_all_items()
+        item_ids = [item.id for item in all_items if item.id is not None]
+        self.delete(item_ids)
+
+    def to_pandas(self) -> pandas.DataFrame:
+        """
+        Convert the dataset to a pandas DataFrame.
+
+        Returns:
+            A pandas DataFrame containing all items in the dataset.
+        """
+        dataset_items = self.get_all_items()
+
+        return converters.to_pandas(dataset_items, keys_mapping={})
+
+    def to_json(self) -> str:
+        """
+        Convert the dataset to a JSON string.
+
+        Returns:
+            A JSON string representation of all items in the dataset.
+        """
+        dataset_items = self.get_all_items()
+
+        return converters.to_json(dataset_items, keys_mapping={})
+
+    def get_all_items(self) -> List[dataset_item.DatasetItem]:
+        """
+        Retrieve all items from the dataset.
+
+        Returns:
+            A list of DatasetItem objects representing all items in the dataset.
+        """
+        results: List[dataset_item.DatasetItem] = []
+
+        while True:
+            stream = self._rest_client.datasets.stream_dataset_items(
+                dataset_name=self._name,
+                last_retrieved_id=results[-1].id if len(results) > 0 else None,
+            )
+
+            item_bytes = b"".join(stream)
+            stream_results: List[dataset_item.DatasetItem] = []
+            for line in item_bytes.split(b"\n"):
+                if len(line) == 0:
+                    continue
+
+                item_content: Dict[str, Any] = json.loads(line.decode("utf-8").strip())
+
+                item = dataset_item.DatasetItem(
+                    id=item_content.get("id"),  # type: ignore
+                    input=item_content.get("input"),  # type: ignore
+                    expected_output=item_content.get("expected_output"),  # type: ignore
+                    metadata=item_content.get("metadata"),  # type: ignore
+                    trace_id=item_content.get("trace_id"),  # type: ignore
+                    span_id=item_content.get("span_id"),  # type: ignore
+                    source=item_content.get("source"),  # type: ignore
+                )
+
+                stream_results.append(item)
+
+            if len(stream_results) == 0:
+                break
+
+            results.extend(stream_results)
+
+        return results
+
+    def insert_from_json(
+        self,
+        json_array: str,
+        keys_mapping: Optional[Dict[str, str]] = None,
+        ignore_keys: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Args:
+            json_array: json string of format: "[{...}, {...}, {...}]" where every dictionary
+                is to be transformed into dataset item
+            keys_mapping: dictionary that maps json keys to item fields names
+                Example: {'Expected output': 'expected_output'}
+            ignore_keys: if your json dicts contain keys that are not needed for DatasetItem
+                construction - pass them as ignore_keys argument
+        """
+        keys_mapping = {} if keys_mapping is None else keys_mapping
+        ignore_keys = [] if ignore_keys is None else ignore_keys
+
+        new_items = converters.from_json(
+            json_array, keys_mapping=keys_mapping, ignore_keys=ignore_keys
+        )
+
+        self.insert(new_items)
+
+    def insert_from_pandas(
+        self,
+        dataframe: pandas.DataFrame,
+        keys_mapping: Optional[Dict[str, str]] = None,
+        ignore_keys: Optional[List[str]] = None,
+    ) -> None:
+        """
+        Args:
+            dataframe: pandas dataframe
+            keys_mapping: Dictionary that maps dataframe column names to dataset item field names.
+                Example: {'Expected output': 'expected_output'}
+            ignore_keys: if your dataframe contains columns that are not needed for DatasetItem
+                construction - pass them as ignore_keys argument
+        """
+        keys_mapping = {} if keys_mapping is None else keys_mapping
+        ignore_keys = [] if ignore_keys is None else ignore_keys
+
+        new_items = converters.from_pandas(dataframe, keys_mapping, ignore_keys)
+
+        self.insert(new_items)

opik-0.1.3/src/opik/api_objects/dataset/dataset_item.py

@@ -0,0 +1,37 @@
+from typing import Optional, Dict, Any
+import pydantic
+from .. import constants
+
+
+class DatasetItem(pydantic.BaseModel):
+    """A DatasetItem object representing an item in a dataset."""
+
+    model_config = pydantic.ConfigDict(strict=True)
+
+    input: Dict[str, Any]
+    """The input data for the dataset item."""
+
+    expected_output: Optional[Dict[str, Any]] = None
+    """The expected output for the dataset item."""
+
+    metadata: Optional[Dict[str, Any]] = None
+    """Additional metadata associated with the dataset item."""
+
+    trace_id: Optional[str] = None
+    """The ID of the trace associated with this dataset item."""
+
+    span_id: Optional[str] = None
+    """The ID of the span associated with this dataset item."""
+
+    id: Optional[str] = None
+    """The unique identifier for this dataset item."""
+
+    source: str = constants.DATASET_SOURCE_SDK
+    """The source of the dataset item. Defaults to DATASET_SOURCE_SDK."""
+
+    def get_content(self) -> Dict[str, Any]:
+        return {
+            "input": self.input,
+            "expected_output": self.expected_output,
+            "metadata": self.metadata,
+        }

opik-0.1.3/src/opik/api_objects/experiment/__init__.py

@@ -0,0 +1,3 @@
+from .experiment import Experiment
+
+__all__ = ["Experiment"]

opik-0.1.3/src/opik/api_objects/experiment/experiment.py

@@ -0,0 +1,45 @@
+import logging
+from typing import List
+
+from opik.rest_api import client as rest_api_client
+from opik.rest_api.types import experiment_item as rest_experiment_item
+from . import experiment_item
+from .. import helpers, constants
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class Experiment:
+    def __init__(
+        self,
+        id: str,
+        name: str,
+        dataset_name: str,
+        rest_client: rest_api_client.OpikApi,
+    ) -> None:
+        self._id = id
+        self._name = name
+        self._dataset_name = dataset_name
+        self._rest_client = rest_client
+
+    def insert(self, experiment_items: List[experiment_item.ExperimentItem]) -> None:
+        rest_experiment_items = [
+            rest_experiment_item.ExperimentItem(
+                id=item.id if item.id is not None else helpers.generate_id(),
+                experiment_id=self._id,
+                dataset_item_id=item.dataset_item_id,
+                trace_id=item.trace_id,
+            )
+            for item in experiment_items
+        ]
+
+        batches = helpers.list_to_batches(
+            rest_experiment_items, batch_size=constants.EXPERIMENT_ITEMS_MAX_BATCH_SIZE
+        )
+
+        for batch in batches:
+            LOGGER.debug("Sending experiment items batch: %s", batch)
+            self._rest_client.experiments.create_experiment_items(
+                experiment_items=batch,
+            )

opik-0.1.3/src/opik/api_objects/experiment/experiment_item.py

@@ -0,0 +1,10 @@
+import dataclasses
+
+from typing import Optional
+
+
+@dataclasses.dataclass
+class ExperimentItem:
+    dataset_item_id: str
+    trace_id: str
+    id: Optional[str] = None

opik-0.1.3/src/opik/api_objects/helpers.py

@@ -0,0 +1,24 @@
+import datetime
+import uuid_extensions
+
+from typing import List, Any
+
+from .. import datetime_helpers
+from typing import Optional
+
+
+def generate_id() -> str:
+    return str(uuid_extensions.uuid7())
+
+
+def datetime_to_iso8601_if_not_None(
+    value: Optional[datetime.datetime],
+) -> Optional[str]:
+    if value is None:
+        return None
+
+    return datetime_helpers.datetime_to_iso8601(value)
+
+
+def list_to_batches(items: List[Any], batch_size: int) -> List[List[Any]]:
+    return [items[i : i + batch_size] for i in range(0, len(items), batch_size)]