wallaroo 2024.3.0.dev0__tar.gz → 2024.3.2__tar.gz

{wallaroo-2024.3.0.dev0 → wallaroo-2024.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: wallaroo
-Version: 2024.3.0.dev0
+Version: 2024.3.2
 Summary: Wallaroo.ai model management API client
 Project-URL: Homepage, https://www.wallaroo.ai/
 Author-email: "Wallaroo.ai" <hello@wallaroo.ai>

{wallaroo-2024.3.0.dev0 → wallaroo-2024.3.2}/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2024.3.0.dev0'
-__version_tuple__ = version_tuple = (2024, 3, 0, 'dev0')
+__version__ = version = '2024.3.2'
+__version_tuple__ = version_tuple = (2024, 3, 2)

{wallaroo-2024.3.0.dev0 → wallaroo-2024.3.2}/mac/__init__.py

@@ -17,12 +17,12 @@ import os
 
 from rich.logging import RichHandler
 
-log_level = os.getenv("LOG_LEVEL", "INFO").upper()
+log_level = os.getenv("LOG_LEVEL", "DEBUG").upper()
 
 logging.basicConfig(
     format="%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s: %(message)s",
     datefmt="%d-%b-%y %H:%M:%S",
-    level=getattr(logging, log_level, logging.INFO),
+    level=getattr(logging, log_level, logging.DEBUG),
     handlers=[RichHandler(show_time=False)],
 )
 

{wallaroo-2024.3.0.dev0 → wallaroo-2024.3.2}/mac/config/service/server_config.py

@@ -25,7 +25,7 @@ class ServerConfig(BaseSettings):
     port: int = 8080
 
     # whether to use a lock to synchronize access to `PythonStep`
-    use_lock: bool = False
+    use_lock: bool = True
 
     @field_validator("host")
     @classmethod

{wallaroo-2024.3.0.dev0 → wallaroo-2024.3.2}/mac/service/arrow_flight/server.py

@@ -62,6 +62,7 @@ class ArrowFlightServer(flight.FlightServerBase):
         self._lock: Optional[threading.Lock] = (
             threading.Lock() if self._server_config.use_lock is True else None
         )
+        logger.debug("Lock is set to %s", self._lock)
         super().__init__(location=self.location)
 
     @property
@@ -133,11 +134,13 @@ class ArrowFlightServer(flight.FlightServerBase):
     @contextmanager
     def _acquire_lock_if_set(self):
         if self._lock:
+            logger.debug("Acquiring lock...")
             self._lock.acquire()
         try:
             yield
         finally:
             if self._lock:
+                logger.debug("Releasing lock...")
                 self._lock.release()
 
     def _run_inference_and_write_to_stream(

{wallaroo-2024.3.0.dev0 → wallaroo-2024.3.2}/pyproject.toml

@@ -81,10 +81,12 @@ artifacts = ["./wallaroo/wallaroo_ml_ops_api_client"]
 [tool.hatch.envs.test]
 env-exclude = ["WALLAROO*"]
 dependencies = [
-  "aioresponses == 0.7.2",
   "httpx == 0.23.0",
-  "pytest == 6.2.4",
-  "pytest-snapshot==0.6.1",
+  "pytest == 8.3.3",
+  "freezegun == 1.5.1",
+  "pytest-mock == 3.14.0",
+  "pytest-snapshot== 0.9.0",
+  "pytest-asyncio == 0.24.0",
   "responses == 0.21.0",
   "respx == 0.19.2",
   # Required for mypy type checking
@@ -95,9 +97,13 @@ dependencies = [
   # Required for linting and formatting
   "ruff == 0.6.2",
   # Required for cov
-  "pytest-cov == 4.0.0",
+  "pytest-cov == 5.0.0",
 ]
 
+#TODO start testing for multiple versions
+[[tool.hatch.envs.test.matrix]]
+python = ["3.8", "3.9"]
+
 # This environment is for Sphinx documentation.
 [tool.hatch.envs.docs]
 dependencies = [
@@ -144,9 +150,9 @@ fix_format = "python -m ruff format wallaroo"
 lint = "python -m ruff check wallaroo"
 fix_lint = "python -m ruff check --fix wallaroo"
 
-# TODO: This doesn't seem to do anything, neither here nor in pytest.ini
 [tool.pytest.ini_options]
-filterwarnings = ['ignore::DeprecationWarning']
+filterwarnings = ['ignore::DeprecationWarning:onnx.*']
+asyncio_default_fixture_loop_scope = "function"
 
 [tool.ruff]
 exclude = [
@@ -172,7 +178,7 @@ exclude = [
     "node_modules",
     "venv",
     "__init__.py",
-    "unit_tests/",
+    "unit_tests/(?!assays_v2/)",
     "wallaroo/wallaroo_ml_ops_api_client",
     "wallaroo/graphql",
     "wallaroo/records"

wallaroo-2024.3.2/unit_tests/assays_v2/test_assay_result_v2.py

@@ -0,0 +1,326 @@
+from datetime import datetime, timezone
+
+import pandas as pd
+import pytest
+
+from wallaroo.assays_v2 import AssayV2
+from wallaroo.assays_v2.assay_result_v2 import AssayResultsList
+
+
+@pytest.fixture
+def assay_results_list(assay_result_v2):
+    return AssayResultsList([assay_result_v2], assay_result_v2.parent_assay)
+
+
+def test_init_assay_result_v2(
+    assay_result_v2, baseline_summaries, mlops_assay_result_v2, assay_window_summaries
+):
+    assert isinstance(assay_result_v2.parent_assay, AssayV2)
+    assert assay_result_v2.raw == mlops_assay_result_v2
+    assert assay_result_v2.v1_iopath == "out.variable.0"
+    assert assay_result_v2._baseline_summary == baseline_summaries["out.variable.0"]
+    assert assay_result_v2._window_summary == assay_window_summaries["out.variable.0"]
+
+
+def test_compare_basic_stats(
+    assay_result_v2,
+    baseline_start_end,
+    baseline_stats,
+    window_start_end,
+    window_stats,
+):
+    data = {
+        "Baseline": [
+            baseline_stats.count,
+            baseline_stats.max_,
+            baseline_stats.mean,
+            baseline_stats.median,
+            baseline_stats.min_,
+            baseline_stats.std,
+            baseline_start_end[0],
+            baseline_start_end[1],
+        ],
+        "Window": [
+            window_stats.count,
+            window_stats.max_,
+            window_stats.mean,
+            window_stats.median,
+            window_stats.min_,
+            window_stats.std,
+            window_start_end[0],
+            window_start_end[1],
+        ],
+        "diff": [500.0, 1.0, 1.0, 1.0, 1.0, 0.0, None, None],
+        "pct_diff": [100.0, 20.0, 33.333333, 33.333333, 100.0, 0.0, None, None],
+    }
+    index = ["count", "max", "mean", "median", "min", "std", "start", "end"]
+    expected_df = pd.DataFrame(data, index=index)
+    expected_df["pct_diff"] = expected_df["pct_diff"].astype("O")
+    expected_df["diff"] = expected_df["diff"].astype("O")
+
+    df = assay_result_v2.compare_basic_stats()
+
+    pd.testing.assert_frame_equal(df.sort_index(axis=1), expected_df.sort_index(axis=1))
+
+
+def test_init_assay_results_list(assay_results_list, assay_result_v2, assay_v2):
+    assert isinstance(assay_results_list, AssayResultsList)
+    assert assay_results_list.parent_assay == assay_v2
+    assert len(assay_results_list) == 1
+    assert assay_results_list[0] == assay_result_v2
+
+
+def test_to_df_row(
+    assay_result_v2,
+    window_start_end,
+    baseline_summaries,
+    assay_window_summaries,
+    targeting,
+    scores,
+    assay_v2_builder,
+):
+    expected_row = {
+        "id": "some-assay-result-id",
+        "assay_id": "some-assay-id",
+        "window_start": datetime.fromisoformat(window_start_end[0]),
+        "analyzed_at": datetime(2023, 1, 1, 1, 0, tzinfo=timezone.utc),
+        "elapsed_millis": 1000,
+        "pipeline_id": 1,
+        "workspace_id": 1,
+        "workspace_name": "some-workspace-name",
+        "baseline_summary": baseline_summaries["out.variable.0"].to_dict(),
+        "window_summary": assay_window_summaries["out.variable.0"].to_dict(),
+        "warning_threshold": targeting.iopath[0].thresholds.warning,
+        "alert_threshold": targeting.iopath[0].thresholds.alert,
+        "bin_index": None,
+        "summarizer": assay_v2_builder.summarizer.to_dict(),
+        "status": "Warning",
+        "created_at": datetime(2023, 1, 1, 1, 0, tzinfo=timezone.utc),
+        "score": scores["out.variable.0"].score,
+        "scores": scores["out.variable.0"].scores,
+    }
+
+    row = assay_result_v2.to_df_row()
+
+    assert row == expected_row
+
+
+def test_to_dataframe(
+    assay_window_summaries,
+    assay_results_list,
+    window_start_end,
+    scores,
+    targeting,
+):
+    expected_df = pd.DataFrame(
+        {
+            "id": ["some-assay-result-id"],
+            "assay_id": ["some-assay-id"],
+            "assay_name": ["some-assay-name"],
+            "iopath": ["out.variable.0"],
+            "pipeline_id": [1],
+            "pipeline_name": ["some-pipeline-name"],
+            "workspace_id": [1],
+            "workspace_name": ["some-workspace-name"],
+            "score": [scores["out.variable.0"].score],
+            "start": [window_start_end[0]],
+            "min": [assay_window_summaries["out.variable.0"].statistics.min_],
+            "max": [assay_window_summaries["out.variable.0"].statistics.max_],
+            "mean": [assay_window_summaries["out.variable.0"].statistics.mean],
+            "median": [assay_window_summaries["out.variable.0"].statistics.median],
+            "std": [assay_window_summaries["out.variable.0"].statistics.std],
+            "status": ["Warning"],
+            "warning_threshold": [targeting.iopath[0].thresholds.warning],
+            "alert_threshold": [targeting.iopath[0].thresholds.alert],
+        }
+    )
+
+    df = assay_results_list.to_dataframe()
+
+    pd.testing.assert_frame_equal(df.sort_index(axis=1), expected_df.sort_index(axis=1))
+
+
+def test_to_full_dataframe(
+    assay_window_summaries,
+    baseline_summaries,
+    assay_results_list,
+    targeting,
+    scores,
+    window_start_end,
+    baseline_start_end,
+):
+    expected_df = pd.DataFrame(
+        {
+            "id": ["some-assay-result-id"],
+            "assay_id": ["some-assay-id"],
+            "window_start": [datetime.fromisoformat(window_start_end[0])],
+            "analyzed_at": [datetime(2023, 1, 1, 1, 0, tzinfo=timezone.utc)],
+            "elapsed_millis": [1000],
+            "pipeline_id": [1],
+            "workspace_id": [1],
+            "workspace_name": ["some-workspace-name"],
+            "baseline_summary_aggregated_values_0": [
+                baseline_summaries["out.variable.0"].aggregated_values[0]
+            ],
+            "baseline_summary_aggregated_values_1": [
+                baseline_summaries["out.variable.0"].aggregated_values[1]
+            ],
+            "baseline_summary_aggregated_values_2": [
+                baseline_summaries["out.variable.0"].aggregated_values[2]
+            ],
+            "baseline_summary_aggregated_values_3": [
+                baseline_summaries["out.variable.0"].aggregated_values[3]
+            ],
+            "baseline_summary_aggregated_values_4": [
+                baseline_summaries["out.variable.0"].aggregated_values[4]
+            ],
+            "baseline_summary_aggregation": ["Cumulative"],
+            "baseline_summary_bins_edges_0": [
+                baseline_summaries["out.variable.0"].bins.edges[0]
+            ],
+            "baseline_summary_bins_edges_1": [
+                baseline_summaries["out.variable.0"].bins.edges[1]
+            ],
+            "baseline_summary_bins_edges_2": [
+                baseline_summaries["out.variable.0"].bins.edges[2]
+            ],
+            "baseline_summary_bins_edges_3": [
+                baseline_summaries["out.variable.0"].bins.edges[3]
+            ],
+            "baseline_summary_bins_edges_4": [
+                baseline_summaries["out.variable.0"].bins.edges[4]
+            ],
+            "baseline_summary_bins_labels_0": [
+                baseline_summaries["out.variable.0"].bins.labels[0]
+            ],
+            "baseline_summary_bins_labels_1": [
+                baseline_summaries["out.variable.0"].bins.labels[1]
+            ],
+            "baseline_summary_bins_labels_2": [
+                baseline_summaries["out.variable.0"].bins.labels[2]
+            ],
+            "baseline_summary_bins_labels_3": [
+                baseline_summaries["out.variable.0"].bins.labels[3]
+            ],
+            "baseline_summary_bins_labels_4": [
+                baseline_summaries["out.variable.0"].bins.labels[4]
+            ],
+            "baseline_summary_bins_mode_Equal": [
+                baseline_summaries["out.variable.0"].bins.mode.equal
+            ],
+            "baseline_summary_name": ["out.variable.0"],
+            "baseline_summary_statistics_count": [
+                baseline_summaries["out.variable.0"].statistics.count
+            ],
+            "baseline_summary_statistics_max": [
+                baseline_summaries["out.variable.0"].statistics.max_
+            ],
+            "baseline_summary_statistics_mean": [
+                baseline_summaries["out.variable.0"].statistics.mean
+            ],
+            "baseline_summary_statistics_median": [
+                baseline_summaries["out.variable.0"].statistics.median
+            ],
+            "baseline_summary_statistics_min": [
+                baseline_summaries["out.variable.0"].statistics.min_
+            ],
+            "baseline_summary_statistics_std": [
+                baseline_summaries["out.variable.0"].statistics.std
+            ],
+            "baseline_summary_end": [baseline_start_end[1]],
+            "baseline_summary_start": [baseline_start_end[0]],
+            "window_summary_aggregated_values_0": [
+                assay_window_summaries["out.variable.0"].aggregated_values[0]
+            ],
+            "window_summary_aggregated_values_1": [
+                assay_window_summaries["out.variable.0"].aggregated_values[1]
+            ],
+            "window_summary_aggregated_values_2": [
+                assay_window_summaries["out.variable.0"].aggregated_values[2]
+            ],
+            "window_summary_aggregated_values_3": [
+                assay_window_summaries["out.variable.0"].aggregated_values[3]
+            ],
+            "window_summary_aggregated_values_4": [
+                assay_window_summaries["out.variable.0"].aggregated_values[4]
+            ],
+            "window_summary_bins_edges_0": [
+                assay_window_summaries["out.variable.0"].bins.edges[0]
+            ],
+            "window_summary_bins_edges_1": [
+                assay_window_summaries["out.variable.0"].bins.edges[1]
+            ],
+            "window_summary_bins_edges_2": [
+                assay_window_summaries["out.variable.0"].bins.edges[2]
+            ],
+            "window_summary_bins_edges_3": [
+                assay_window_summaries["out.variable.0"].bins.edges[3]
+            ],
+            "window_summary_bins_edges_4": [
+                assay_window_summaries["out.variable.0"].bins.edges[4]
+            ],
+            "window_summary_bins_labels_0": [
+                assay_window_summaries["out.variable.0"].bins.labels[0]
+            ],
+            "window_summary_bins_labels_1": [
+                assay_window_summaries["out.variable.0"].bins.labels[1]
+            ],
+            "window_summary_bins_labels_2": [
+                assay_window_summaries["out.variable.0"].bins.labels[2]
+            ],
+            "window_summary_bins_labels_3": [
+                assay_window_summaries["out.variable.0"].bins.labels[3]
+            ],
+            "window_summary_bins_labels_4": [
+                assay_window_summaries["out.variable.0"].bins.labels[4]
+            ],
+            "window_summary_bins_mode_Equal": [
+                assay_window_summaries["out.variable.0"].bins.mode.equal
+            ],
+            "window_summary_statistics_count": [
+                assay_window_summaries["out.variable.0"].statistics.count
+            ],
+            "window_summary_statistics_max": [
+                assay_window_summaries["out.variable.0"].statistics.max_
+            ],
+            "window_summary_statistics_mean": [
+                assay_window_summaries["out.variable.0"].statistics.mean
+            ],
+            "window_summary_statistics_median": [
+                assay_window_summaries["out.variable.0"].statistics.median
+            ],
+            "window_summary_statistics_min": [
+                assay_window_summaries["out.variable.0"].statistics.min_
+            ],
+            "window_summary_statistics_std": [
+                assay_window_summaries["out.variable.0"].statistics.std
+            ],
+            "window_summary_end": [window_start_end[1]],
+            "window_summary_start": [window_start_end[0]],
+            "warning_threshold": [targeting.iopath[0].thresholds.warning],
+            "alert_threshold": [targeting.iopath[0].thresholds.alert],
+            "score": [scores["out.variable.0"].score],
+            "scores_0": [scores["out.variable.0"].scores[0]],
+            "scores_1": [scores["out.variable.0"].scores[1]],
+            "scores_2": [scores["out.variable.0"].scores[2]],
+            "scores_3": [scores["out.variable.0"].scores[3]],
+            "scores_4": [scores["out.variable.0"].scores[4]],
+            "bin_index": [None],
+            "summarizer_UnivariateContinuous_aggregation": [
+                baseline_summaries["out.variable.0"].aggregation
+            ],
+            "summarizer_UnivariateContinuous_bin_mode_Equal": [
+                baseline_summaries["out.variable.0"].bins.mode.equal
+            ],
+            "summarizer_UnivariateContinuous_metric": ["MaxDiff"],
+            "summarizer_UnivariateContinuous_bin_weights": [None],
+            "status": ["Warning"],
+            "created_at": [datetime(2023, 1, 1, 1, 0, tzinfo=timezone.utc)],
+            "window_summary_name": ["out.variable.0"],
+            "window_summary_aggregation": ["Cumulative"],
+        }
+    )
+
+    df = assay_results_list.to_full_dataframe()
+
+    pd.testing.assert_frame_equal(df.sort_index(axis=1), expected_df.sort_index(axis=1))

wallaroo-2024.3.2/unit_tests/assays_v2/test_assay_v2.py

@@ -0,0 +1,165 @@
+from datetime import datetime, timezone
+
+import pytest
+
+from wallaroo.assays_v2 import AssayV2, AssayV2Builder
+from wallaroo.assays_v2.assay_result_v2 import AssayResultsList
+from wallaroo.wallaroo_ml_ops_api_client.api.assays.get_results import (
+    GetResultsBody,
+)
+
+
+@pytest.fixture
+def success_response(mocker):
+    mock_return = mocker.Mock()
+    mock_return.status_code = 200
+    return mock_return
+
+
+@pytest.fixture
+def error_response(mocker):
+    mock_return = mocker.Mock()
+    mock_return.status_code = 400
+    return mock_return
+
+
+def test_init(assay_v2, client):
+    assert isinstance(assay_v2, AssayV2)
+    assert assay_v2.id == "some-assay-id"
+    assert assay_v2._client == client
+
+
+def test_results(assay_v2, mocker, mlops_assay_result_v2):
+    start, end = datetime.now(tz=timezone.utc), datetime.now(tz=timezone.utc)
+    workspace_id = 1
+    mock_return = mocker.Mock()
+    mock_return.parsed = [mlops_assay_result_v2]
+    mock_sync = mocker.patch(
+        "wallaroo.assays_v2.assay_v2.sync_detailed_results",
+        return_value=mock_return,
+    )
+    mock_results_body = mocker.spy(GetResultsBody, "__init__")
+
+    results = assay_v2.results(
+        start=start,
+        end=end,
+        include_failures=True,
+        workspace_id=1,
+    )
+
+    mock_results_body.assert_called_once_with(
+        mocker.ANY, start=start, end=end, id=assay_v2.id, workspace_id=workspace_id
+    )
+    mock_sync.assert_called_once()
+    assert mock_sync.call_args[1]["client"] == assay_v2._client.mlops()
+    assert mock_sync.call_args[1]["body"].id == assay_v2.id
+    assert mock_sync.call_args[1]["body"].start == start
+    assert mock_sync.call_args[1]["body"].end == end
+    assert mock_sync.call_args[1]["body"].workspace_id == workspace_id
+    assert isinstance(results, AssayResultsList)
+    assert len(results) == 1
+    assert results[0].raw == mlops_assay_result_v2
+
+
+def test_set_active(assay_v2, mocker):
+    mock_return = mocker.Mock()
+    mock_return.status_code = 200
+    mock_sync = mocker.patch(
+        "wallaroo.assays_v2.assay_v2.sync_detailed_set_active",
+        return_value=mock_return,
+    )
+
+    assay_v2.set_active(True)
+
+    mock_sync.assert_called_once()
+    assert mock_sync.call_args[1]["client"] == assay_v2._client.mlops()
+    assert mock_sync.call_args[1]["body"].active is True
+    assert mock_sync.call_args[1]["body"].assay_id == assay_v2.id
+
+
+@pytest.mark.parametrize(
+    "active, expected_verb",
+    [(True, "resume"), (False, "pause"), ("some-value", "pause")],
+)
+def test_set_active_raise_error(
+    assay_v2, mocker, active, expected_verb, error_response
+):
+    mocker.patch(
+        "wallaroo.assays_v2.assay_v2.sync_detailed_set_active",
+        return_value=error_response,
+    )
+
+    with pytest.raises(Exception, match=f"Failed to {expected_verb} assay."):
+        assay_v2.set_active(active)
+
+
+def test_pause(assay_v2, mocker, success_response):
+    mock_sync = mocker.patch(
+        "wallaroo.assays_v2.assay_v2.sync_detailed_set_active",
+        return_value=success_response,
+    )
+
+    assay_v2.pause()
+
+    mock_sync.assert_called_once()
+    assert mock_sync.call_args[1]["client"] == assay_v2._client.mlops()
+    assert mock_sync.call_args[1]["body"].active is False
+    assert mock_sync.call_args[1]["body"].assay_id == assay_v2.id
+
+
+def test_resume(assay_v2, mocker, success_response):
+    mock_sync = mocker.patch(
+        "wallaroo.assays_v2.assay_v2.sync_detailed_set_active",
+        return_value=success_response,
+    )
+
+    assay_v2.resume()
+
+    mock_sync.assert_called_once()
+    assert mock_sync.call_args[1]["client"] == assay_v2._client.mlops()
+    assert mock_sync.call_args[1]["body"].active is True
+    assert mock_sync.call_args[1]["body"].assay_id == assay_v2.id
+
+
+def test_builder(client, mocker):
+    mocker.patch(
+        "wallaroo.workspace.Workspace.name",
+        return_value="some-workspace-name",
+    )
+    builder = AssayV2.builder(client, 1, "some-pipeline-name", 1)
+    assert isinstance(builder, AssayV2Builder)
+    assert builder.client == client
+    assert builder.name == "some-pipeline-name assay"
+    assert builder.targeting.data_origin.pipeline_id == 1
+    assert builder.targeting.data_origin.pipeline_name == "some-pipeline-name"
+    assert builder.targeting.data_origin.workspace_id == 1
+
+
+def test_next_run(assay_v2, mocker):
+    mock_return = mocker.Mock()
+    mock_return.parsed = "some-value"
+
+    mocker.patch(
+        "wallaroo.wallaroo_ml_ops_api_client.api.assays.get_next_run.sync_detailed",
+        return_value=mock_return,
+    )
+
+    next_run = assay_v2._next_run()
+    assert next_run == "some-value"
+
+
+def test_next_run_raise_error(assay_v2, mocker):
+    mock_return = mocker.Mock()
+    mock_return.parsed = None
+    mock_return.content = "some-error"
+    mocker.patch(
+        "wallaroo.wallaroo_ml_ops_api_client.api.assays.get_next_run.sync_detailed",
+        return_value=mock_return,
+    )
+
+    with pytest.raises(Exception, match="some-error"):
+        assay_v2._next_run()
+
+
+def test_get_iopath(assay_v2):
+    assert assay_v2._get_iopath() == "out.variable.0"