orca-sdk 0.1.9__py3-none-any.whl

orca_sdk/telemetry_test.py

@@ -0,0 +1,119 @@
+import pytest
+
+from .classification_model import ClassificationModel
+from .memoryset import LabeledMemoryLookup
+from .telemetry import ClassificationPrediction, FeedbackCategory
+
+
+def test_get_prediction(classification_model: ClassificationModel):
+    predictions = classification_model.predict(["Do you love soup?", "Are cats cute?"])
+    assert len(predictions) == 2
+    assert predictions[0].prediction_id is not None
+    assert predictions[1].prediction_id is not None
+    prediction_with_telemetry = ClassificationPrediction.get(predictions[0].prediction_id)
+    assert prediction_with_telemetry is not None
+    assert prediction_with_telemetry.label == 0
+    assert prediction_with_telemetry.input_value == "Do you love soup?"
+
+
+def test_get_predictions(classification_model: ClassificationModel):
+    predictions = classification_model.predict(["Do you love soup?", "Are cats cute?"])
+    assert len(predictions) == 2
+    assert predictions[0].prediction_id is not None
+    assert predictions[1].prediction_id is not None
+    prediction_with_telemetry = ClassificationPrediction.get(
+        [predictions[0].prediction_id, predictions[1].prediction_id]
+    )
+    assert len(prediction_with_telemetry) == 2
+    assert prediction_with_telemetry[0].label == 0
+    assert prediction_with_telemetry[0].input_value == "Do you love soup?"
+    assert prediction_with_telemetry[1].label == 1
+
+
+def test_get_predictions_with_expected_label_match(classification_model: ClassificationModel):
+    classification_model.predict(
+        ["Do you love soup?", "Are cats cute?"], expected_labels=[0, 0], tags={"expected_label_match"}
+    )
+    classification_model.predict("no expectations", tags={"expected_label_match"})
+    assert len(classification_model.predictions(tag="expected_label_match")) == 3
+    assert len(classification_model.predictions(expected_label_match=True, tag="expected_label_match")) == 1
+    assert len(classification_model.predictions(expected_label_match=False, tag="expected_label_match")) == 1
+
+
+def test_get_prediction_memory_lookups(classification_model: ClassificationModel):
+    prediction = classification_model.predict("Do you love soup?")
+    assert isinstance(prediction.memory_lookups, list)
+    assert len(prediction.memory_lookups) > 0
+    assert all(isinstance(lookup, LabeledMemoryLookup) for lookup in prediction.memory_lookups)
+
+
+def test_record_feedback(classification_model: ClassificationModel):
+    prediction = classification_model.predict("Do you love soup?")
+    assert "correct" not in prediction.feedback
+    prediction.record_feedback(category="correct", value=prediction.label == 0)
+    assert prediction.feedback["correct"] is True
+
+
+def test_record_feedback_with_invalid_value(classification_model: ClassificationModel):
+    with pytest.raises(ValueError, match=r"Invalid input.*"):
+        classification_model.predict("Do you love soup?").record_feedback(category="correct", value="not a bool")  # type: ignore
+
+
+def test_record_feedback_with_inconsistent_value_for_category(classification_model: ClassificationModel):
+    classification_model.predict("Do you love soup?").record_feedback(category="correct", value=True)
+    with pytest.raises(ValueError, match=r"Invalid input.*"):
+        classification_model.predict("Do you love soup?").record_feedback(category="correct", value=-1.0)
+
+
+def test_delete_feedback(classification_model: ClassificationModel):
+    prediction = classification_model.predict("Do you love soup?")
+    prediction.record_feedback(category="test_delete", value=True)
+    assert "test_delete" in prediction.feedback
+    prediction.delete_feedback("test_delete")
+    assert "test_delete" not in prediction.feedback
+
+
+def test_list_feedback_categories(classification_model: ClassificationModel):
+    prediction = classification_model.predict("Do you love soup?")
+    prediction.record_feedback(category="correct", value=True)
+    prediction.record_feedback(category="confidence", value=0.8)
+    categories = FeedbackCategory.all()
+    assert len(categories) >= 2
+    assert any(c.name == "correct" and c.value_type == bool for c in categories)
+    assert any(c.name == "confidence" and c.value_type == float for c in categories)
+
+
+def test_drop_feedback_category(classification_model: ClassificationModel):
+    prediction = classification_model.predict("Do you love soup?")
+    prediction.record_feedback(category="test_category", value=True)
+    assert any(c.name == "test_category" for c in FeedbackCategory.all())
+    FeedbackCategory.drop("test_category")
+    assert not any(c.name == "test_category" for c in FeedbackCategory.all())
+    prediction.refresh()
+    assert "test_category" not in prediction.feedback
+
+
+def test_update_prediction(classification_model: ClassificationModel):
+    prediction = classification_model.predict("Do you love soup?")
+    assert prediction.expected_label is None
+    assert prediction.tags == set()
+    # update expected label
+    prediction.update(expected_label=1)
+    assert prediction.expected_label == 1
+
+    # update tags
+    prediction.update(tags={"test_tag1", "test_tag2"})
+    assert prediction.tags == {"test_tag1", "test_tag2"}
+
+    # update both
+    prediction.update(expected_label=0, tags={"new_tag"})
+    assert prediction.expected_label == 0
+    assert prediction.tags == {"new_tag"}
+
+    # remove expected label
+    prediction.update(expected_label=None)
+    assert prediction.expected_label is None
+
+    # remove tags
+    prediction.update(tags=None)
+    assert prediction.tags == set()

orca_sdk-0.1.9.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: orca_sdk
+Version: 0.1.9
+Summary: SDK for interacting with Orca Services
+License-Expression: Apache-2.0
+Author: Orca DB Inc.
+Author-email: dev-rel@orcadb.ai
+Requires-Python: >=3.11,<3.15
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: datasets (>=4.4.0,<5)
+Requires-Dist: gradio (>=6.0.0,<7)
+Requires-Dist: httpx (>=0.28.1)
+Requires-Dist: httpx-retries (>=0.4.3,<0.5.0)
+Requires-Dist: numpy (>=2.1.0,<3)
+Requires-Dist: pandas (>=2.2.3,<3)
+Requires-Dist: pyarrow (>=22.0.0,<23)
+Requires-Dist: python-dotenv (>=1.1.0)
+Requires-Dist: scikit-learn (>=1.6.1,<2)
+Requires-Dist: torch (>=2.8.0,<3)
+Description-Content-Type: text/markdown
+
+<!--
+IMPORTANT NOTE:
+- This file will get rendered in the public facing PyPi page here: https://pypi.org/project/orca_sdk/
+- Only content suitable for public consumption should be placed in this file everything else should go into CONTRIBUTING.md
+-->
+
+# OrcaSDK
+
+OrcaSDK is a Python library for building and using retrieval-augmented models with [OrcaCloud](https://orcadb.ai). It enables you to create, deploy, and maintain models that can adapt to changing circumstances without retraining by accessing external data called "memories."
+
+## Documentation
+
+You can find the documentation for all things Orca at [docs.orcadb.ai](https://docs.orcadb.ai). This includes tutorials, how-to guides, and the full interface reference for OrcaSDK.
+
+## Features
+
+- **Labeled Memorysets**: Store and manage labeled examples that your models can use to guide predictions
+- **Classification Models**: Build retrieval-augmented classification models that adapt to new data without retraining
+- **Embedding Models**: Use pre-trained or fine-tuned embedding models to represent your data
+- **Telemetry**: Collect feedback and monitor memory usage to optimize model performance
+- **Datasources**: Easily ingest data from various sources into your memorysets
+
+## Installation
+
+OrcaSDK is compatible with Python 3.10 or higher and is available on [PyPI](https://pypi.org/project/orca_sdk/). You can install it with your favorite python package manager:
+
+- Pip: `pip install orca_sdk`
+- Conda: `conda install orca_sdk`
+- Poetry: `poetry add orca_sdk`
+
+## Quick Start
+
+```python
+from dotenv import load_dotenv
+from orca_sdk import OrcaCredentials, LabeledMemoryset, ClassificationModel
+
+# Load your API key from environment variables
+load_dotenv()
+assert OrcaCredentials.is_authenticated()
+
+# Create a labeled memoryset
+memoryset = LabeledMemoryset.from_disk("my_memoryset", "./data.jsonl")
+
+# Create a classification model using the memoryset
+model = ClassificationModel("my_model", memoryset)
+
+# Make predictions
+prediction = model.predict("my input")
+
+# Get Action Recommendation
+action, rationale = prediction.recommend_action()
+print(f"Recommended action: {action}")
+print(f"Rationale: {rationale}")
+
+# Generate and add synthetic memory suggestions
+if action == "add_memories":
+    suggestions = prediction.generate_memory_suggestions(num_memories=3)
+
+    # Review suggestions
+    for suggestion in suggestions:
+        print(f"Suggested: '{suggestion['value']}' -> {suggestion['label']}")
+
+    # Add suggestions to memoryset
+    model.memoryset.insert(suggestions)
+    print(f"Added {len(suggestions)} new memories to improve model performance!")
+```
+
+For a more detailed walkthrough, check out our [Quick Start Guide](https://docs.orcadb.ai/quickstart-sdk/).
+
+## Support
+
+If you have any questions, please reach out to us at support@orcadb.ai.
+

orca_sdk-0.1.9.dist-info/RECORD

@@ -0,0 +1,41 @@
+orca_sdk/__init__.py,sha256=xyjNwkLQXaX8A-UYgGwYDjv2btOXArT_yiMTfmW7KA8,1003
+orca_sdk/_shared/__init__.py,sha256=3Kt0Hu3QLI5FEp9nqGTxqAm3hAoBJKcagfaGQZ-lbJQ,223
+orca_sdk/_shared/metrics.py,sha256=faeL1B1ftmns1ikfKrIlU3xOn6j0iAGLNUupxvAFza8,24968
+orca_sdk/_shared/metrics_test.py,sha256=vDIXoj8EuuLcdPJz_7EiVPgQ-FXiVT81JG30jxsg9HM,20752
+orca_sdk/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+orca_sdk/_utils/analysis_ui.py,sha256=R0xc4RyJKyBHJEEF_ztI4Dm5w8Y1uF0Wpkn4LQgXqBE,9258
+orca_sdk/_utils/analysis_ui_style.css,sha256=q_ba_-_KtgztepHg829zLzypaxKayl7ySC1-oYDzV3k,836
+orca_sdk/_utils/auth.py,sha256=nC252O171_3_wn4KBAN7kg8GNvoZFiQ5Xtzkrm5dWDo,2645
+orca_sdk/_utils/auth_test.py,sha256=ygVWv1Ex53LaxIP7p2hzPHl8l9qYyBD5IGmEFJMps6s,1056
+orca_sdk/_utils/common.py,sha256=wUm2pNDWytEecC5WiDWd02-yCZw3Akx0bIutG4lHsFA,805
+orca_sdk/_utils/data_parsing.py,sha256=5vaTpvUOS-ldlcgnSARYw7s9mce-imzkU7kA48-pdIM,5396
+orca_sdk/_utils/data_parsing_test.py,sha256=u7BEjxtsU9gMs3tAZI0lJ--vOLlwKwH3hemdCedzxA0,8826
+orca_sdk/_utils/pagination.py,sha256=986z0QPZixrZeurJWorF6eMgnTRdDF84AagEA6qNbMw,4245
+orca_sdk/_utils/pagination_test.py,sha256=BUylCrcHnwoKEBmMUzVr0lwLpA35ivcCwdBK4rMw9y8,4887
+orca_sdk/_utils/prediction_result_ui.css,sha256=sqBlkRLnovb5X5EcUDdB6iGpH63nVRlTW4uAmXuD0WM,258
+orca_sdk/_utils/prediction_result_ui.py,sha256=Ur_FY7dz3oWNmtPiP3Wl3yRlEMgK8q9UfT-SDu9UPxA,4805
+orca_sdk/_utils/tqdm_file_reader.py,sha256=Lw7Cg1UgNuRUoN6jjqZb-IlV00H-kbRcrZLdudr1GxE,324
+orca_sdk/_utils/value_parser.py,sha256=c3qMABCCDQcIjn9N1orYYnlRwDW9JWdGwW_2TDZPLdI,1286
+orca_sdk/_utils/value_parser_test.py,sha256=OybsiC-Obi32RRi9NIuwrVBRAnlyPMV1xVAaevSrb7M,1079
+orca_sdk/async_client.py,sha256=PM7N-ggmtucfcUF1vQGtTZOCJpSNTOgd7l3LDNF5kP4,137192
+orca_sdk/classification_model.py,sha256=C58euWnNvwXnthR9RtVVCOcgPEbxCjjp3sHMb86V6YA,50197
+orca_sdk/classification_model_test.py,sha256=ElqxtR6gNwwk8dNXwfwAhpT7l0ZIP3H4pHmOyFXyTWk,37370
+orca_sdk/client.py,sha256=SKZv3zGG6OwLe_FlB5wL2cxltOLPCcHvoo2CbMwyKgA,136241
+orca_sdk/conftest.py,sha256=0O1VY-SPKNAvi9fBLdY1RMnYVgZvMjP92y99bNAqqiw,12461
+orca_sdk/credentials.py,sha256=80_1r8n5jruEvN_E629SaRrRhKvF_NhWUEZyZzPXkqQ,6620
+orca_sdk/credentials_test.py,sha256=TLbXJMz3IlThvtSrHeLM7jRsKnrncA_ahOTpHg15Ei4,4089
+orca_sdk/datasource.py,sha256=Qn5QloE84UXeyPk2wcy1lWe5wmh1iDBS044eWnxck_E,22371
+orca_sdk/datasource_test.py,sha256=sCk3IcQJbDut5oN4Wf7PXhTxyMwalxMuCXJekSxy9wk,16665
+orca_sdk/embedding_model.py,sha256=vLGnlO9I-cN1lklNBl_LxZ8m9oK3vkegFOpvYYw8u8g,28038
+orca_sdk/embedding_model_test.py,sha256=Lc6fZ0ifT0hh6ldkUfjwMPcP6OgN0Umlzu8XDLs7UO4,8144
+orca_sdk/job.py,sha256=wHwVt-s7i-v8udhLGybB-90Kp4dwOLrY806bE4Tam5Q,13092
+orca_sdk/job_test.py,sha256=nRSWxd_1UIfrj9oMVvrXjt6OBkBpddYAjb2y6P-DTUg,4327
+orca_sdk/memoryset.py,sha256=06v34fHabpkEaOv9VCKc0NhpMi_mNZGbQP_9GiW_nuE,157157
+orca_sdk/memoryset_test.py,sha256=O2o42XETtffXtZy0kbLk2b8cUDXU-w2ZAzXLi5-vDPQ,51278
+orca_sdk/regression_model.py,sha256=AXRzJG15sDJQSiDCDfRdcLnZDNkJWORYjhHqKyyL-Fc,33960
+orca_sdk/regression_model_test.py,sha256=90EyrhaMk1kTf87RFkMNz1PTItmeUISs6AvHmyp08DU,25447
+orca_sdk/telemetry.py,sha256=ZyCMiyyo_SchjadWZH55TlLrC4Ucq5S316NbW26LL4Y,27834
+orca_sdk/telemetry_test.py,sha256=eT66C5lFdNg-pQdo2I__BP7Tn5fTc9aTkVo9ZhWwhU0,5519
+orca_sdk-0.1.9.dist-info/METADATA,sha256=gmSVjoc0Qa4eo2iotXB3bRmWFYqiehGd_7z67YDHKEw,3709
+orca_sdk-0.1.9.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+orca_sdk-0.1.9.dist-info/RECORD,,

orca_sdk-0.1.9.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.2.1
+Root-Is-Purelib: true
+Tag: py3-none-any