vectorwave 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vectorwave-0.2.0/src/vectorwave.egg-info → vectorwave-0.2.2}/PKG-INFO +1 -1
- {vectorwave-0.2.0 → vectorwave-0.2.2}/pyproject.toml +1 -1
- vectorwave-0.2.2/src/tests/core/llm/test_token_usage.py +112 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/core/test_semantic_caching.py +32 -16
- vectorwave-0.2.2/src/tests/database/test_dataset.py +97 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/utils/test_replayer.py +91 -85
- vectorwave-0.2.2/src/tests/utils/test_return_caching.py +213 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/__init__.py +3 -1
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/core/generator.py +11 -14
- vectorwave-0.2.2/src/vectorwave/core/llm/base.py +47 -0
- vectorwave-0.2.2/src/vectorwave/core/llm/factory.py +13 -0
- vectorwave-0.2.2/src/vectorwave/core/llm/openai_client.py +79 -0
- vectorwave-0.2.2/src/vectorwave/database/dataset.py +150 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/database/db.py +68 -1
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/database/db_search.py +33 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/models/db_config.py +4 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/monitoring/tracer.py +6 -11
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/search/rag_search.py +10 -17
- vectorwave-0.2.2/src/vectorwave/utils/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/utils/healer.py +14 -14
- vectorwave-0.2.2/src/vectorwave/utils/replayer.py +257 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/utils/replayer_semantic.py +44 -48
- vectorwave-0.2.2/src/vectorwave/utils/return_caching_utils.py +152 -0
- vectorwave-0.2.2/src/vectorwave/vectorizer/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/vectorizer/openai_vectorizer.py +16 -6
- {vectorwave-0.2.0 → vectorwave-0.2.2/src/vectorwave.egg-info}/PKG-INFO +1 -1
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave.egg-info/SOURCES.txt +8 -0
- vectorwave-0.2.0/src/tests/utils/test_return_caching.py +0 -126
- vectorwave-0.2.0/src/vectorwave/utils/replayer.py +0 -211
- vectorwave-0.2.0/src/vectorwave/utils/return_caching_utils.py +0 -134
- {vectorwave-0.2.0 → vectorwave-0.2.2}/LICENSE +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/MANIFEST.in +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/NOTICE +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/Readme.md +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/setup.cfg +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/batch/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/batch/test_batch.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/conftest.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/core/__init__.py +0 -0
- {vectorwave-0.2.0/src/tests/database → vectorwave-0.2.2/src/tests/core/llm}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/core/test_decorator.py +0 -0
- {vectorwave-0.2.0/src/tests/exception → vectorwave-0.2.2/src/tests/database}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/database/test_archiver.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/database/test_db.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/database/test_db_search.py +0 -0
- {vectorwave-0.2.0/src/tests/models → vectorwave-0.2.2/src/tests/exception}/__init__.py +0 -0
- {vectorwave-0.2.0/src/tests/monitoring → vectorwave-0.2.2/src/tests/models}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/models/test_db_config.py +0 -0
- {vectorwave-0.2.0/src/tests/monitoring/alert → vectorwave-0.2.2/src/tests/monitoring}/__init__.py +0 -0
- {vectorwave-0.2.0/src/tests/prediction → vectorwave-0.2.2/src/tests/monitoring/alert}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/monitoring/alert/test_alerter.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/monitoring/test_async_trace.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/monitoring/test_tracer.py +0 -0
- {vectorwave-0.2.0/src/tests/search → vectorwave-0.2.2/src/tests/prediction}/__init__.py +0 -0
- {vectorwave-0.2.0/src/tests/utils → vectorwave-0.2.2/src/tests/search}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/search/test_execution_search.py +0 -0
- {vectorwave-0.2.0/src/tests/vectorizer → vectorwave-0.2.2/src/tests/utils}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/utils/test_function_cahe.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/tests/utils/test_status.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/batch → vectorwave-0.2.2/src/tests/vectorizer}/__init__.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/core → vectorwave-0.2.2/src/vectorwave/batch}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/batch/batch.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/database → vectorwave-0.2.2/src/vectorwave/core}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/core/core.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/core/decorator.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/exception → vectorwave-0.2.2/src/vectorwave/core/llm}/__init__.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/models → vectorwave-0.2.2/src/vectorwave/database}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/database/archiver.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/monitoring → vectorwave-0.2.2/src/vectorwave/exception}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/exception/exceptions.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/monitoring/alert → vectorwave-0.2.2/src/vectorwave/models}/__init__.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/prediction → vectorwave-0.2.2/src/vectorwave/monitoring}/__init__.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/search → vectorwave-0.2.2/src/vectorwave/monitoring/alert}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/monitoring/alert/base.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/monitoring/alert/factory.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/monitoring/alert/null_alerter.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/monitoring/alert/webhook_alerter.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/monitoring/monitoring.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/utils → vectorwave-0.2.2/src/vectorwave/prediction}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/prediction/predictor.py +0 -0
- {vectorwave-0.2.0/src/vectorwave/vectorizer → vectorwave-0.2.2/src/vectorwave/search}/__init__.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/search/execution_search.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/search/extended_search.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/utils/function_cache.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/utils/status.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/vectorizer/base.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/vectorizer/factory.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave/vectorizer/huggingface_vectorizer.py +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave.egg-info/dependency_links.txt +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave.egg-info/requires.txt +0 -0
- {vectorwave-0.2.0 → vectorwave-0.2.2}/src/vectorwave.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from unittest.mock import MagicMock, patch
|
|
3
|
+
from vectorwave.core.llm.openai_client import VectorWaveOpenAIClient
|
|
4
|
+
|
|
5
|
+
@pytest.fixture
|
|
6
|
+
def mock_deps(monkeypatch):
|
|
7
|
+
"""
|
|
8
|
+
Mocks the dependencies (BatchManager, OpenAI, Settings) of VectorWaveOpenAIClient.
|
|
9
|
+
"""
|
|
10
|
+
# 1. Mock BatchManager (Target for verifying token usage persistence)
|
|
11
|
+
mock_batch = MagicMock()
|
|
12
|
+
mock_batch.add_object = MagicMock()
|
|
13
|
+
mock_get_batch = MagicMock(return_value=mock_batch)
|
|
14
|
+
monkeypatch.setattr("vectorwave.core.llm.openai_client.get_batch_manager", mock_get_batch)
|
|
15
|
+
|
|
16
|
+
# 2. Mock Settings (Bypasses API Key check)
|
|
17
|
+
mock_settings = MagicMock()
|
|
18
|
+
mock_settings.OPENAI_API_KEY = "sk-test-key"
|
|
19
|
+
mock_get_settings = MagicMock(return_value=mock_settings)
|
|
20
|
+
monkeypatch.setattr("vectorwave.core.llm.openai_client.get_weaviate_settings", mock_get_settings)
|
|
21
|
+
|
|
22
|
+
# 3. Mock OpenAI Class (Bypasses actual API calls)
|
|
23
|
+
mock_openai_cls = MagicMock()
|
|
24
|
+
monkeypatch.setattr("vectorwave.core.llm.openai_client.OpenAI", mock_openai_cls)
|
|
25
|
+
|
|
26
|
+
# Note: Clearing the lru_cache for the singleton instance is typically
|
|
27
|
+
# needed if the client is imported globally. We assume the current test
|
|
28
|
+
# setup allows for fresh initialization here.
|
|
29
|
+
|
|
30
|
+
return {
|
|
31
|
+
"batch": mock_batch,
|
|
32
|
+
"openai_cls": mock_openai_cls
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
def test_chat_completion_logs_token_usage(mock_deps):
|
|
36
|
+
"""
|
|
37
|
+
[Case 1] Verify that token usage is logged as 'generation' type during chat completion.
|
|
38
|
+
"""
|
|
39
|
+
# Arrange
|
|
40
|
+
# Get the mock client instance returned when OpenAI() is called
|
|
41
|
+
mock_client_instance = mock_deps["openai_cls"].return_value
|
|
42
|
+
|
|
43
|
+
# Mock OpenAI Response structure
|
|
44
|
+
mock_response = MagicMock()
|
|
45
|
+
mock_response.choices = [MagicMock(message=MagicMock(content="Test response"))]
|
|
46
|
+
mock_response.usage.total_tokens = 123 # Token count to test
|
|
47
|
+
mock_client_instance.chat.completions.create.return_value = mock_response
|
|
48
|
+
|
|
49
|
+
# Act
|
|
50
|
+
# Create a new client instance for the test
|
|
51
|
+
client = VectorWaveOpenAIClient()
|
|
52
|
+
result = client.create_chat_completion(
|
|
53
|
+
messages=[{"role": "user", "content": "Hi"}],
|
|
54
|
+
model="gpt-4-test",
|
|
55
|
+
category="test_chat_category"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# Assert
|
|
59
|
+
# 1. Verify that the function returns the expected result
|
|
60
|
+
assert result == "Test response"
|
|
61
|
+
|
|
62
|
+
# 2. Verify that BatchManager.add_object was called (log saved)
|
|
63
|
+
mock_batch = mock_deps["batch"]
|
|
64
|
+
mock_batch.add_object.assert_called_once()
|
|
65
|
+
|
|
66
|
+
# 3. Verify the properties of the saved log entry
|
|
67
|
+
call_kwargs = mock_batch.add_object.call_args.kwargs
|
|
68
|
+
props = call_kwargs["properties"]
|
|
69
|
+
|
|
70
|
+
assert call_kwargs["collection"] == "VectorWaveTokenUsage"
|
|
71
|
+
assert props["tokens"] == 123
|
|
72
|
+
assert props["model"] == "gpt-4-test"
|
|
73
|
+
assert props["category"] == "test_chat_category"
|
|
74
|
+
assert props["usage_type"] == "generation"
|
|
75
|
+
|
|
76
|
+
def test_create_embedding_logs_token_usage(mock_deps):
|
|
77
|
+
"""
|
|
78
|
+
[Case 2] Verify that token usage is logged as 'embedding' type during embedding creation.
|
|
79
|
+
"""
|
|
80
|
+
# Arrange
|
|
81
|
+
mock_client_instance = mock_deps["openai_cls"].return_value
|
|
82
|
+
|
|
83
|
+
# Mock OpenAI Response structure
|
|
84
|
+
mock_response = MagicMock()
|
|
85
|
+
mock_response.data = [MagicMock(embedding=[0.1, 0.2, 0.3])]
|
|
86
|
+
mock_response.usage.total_tokens = 45 # Token count to test
|
|
87
|
+
mock_client_instance.embeddings.create.return_value = mock_response
|
|
88
|
+
|
|
89
|
+
# Act
|
|
90
|
+
client = VectorWaveOpenAIClient()
|
|
91
|
+
result = client.create_embedding(
|
|
92
|
+
text="Test text",
|
|
93
|
+
model="text-embedding-3-small",
|
|
94
|
+
category="test_embed_category"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# Assert
|
|
98
|
+
# 1. Verify that the function returns the expected result
|
|
99
|
+
assert result == [0.1, 0.2, 0.3]
|
|
100
|
+
|
|
101
|
+
# 2. Verify that BatchManager.add_object was called
|
|
102
|
+
mock_batch = mock_deps["batch"]
|
|
103
|
+
mock_batch.add_object.assert_called_once()
|
|
104
|
+
|
|
105
|
+
# 3. Verify the properties of the saved log entry
|
|
106
|
+
call_kwargs = mock_batch.add_object.call_args.kwargs
|
|
107
|
+
props = call_kwargs["properties"]
|
|
108
|
+
|
|
109
|
+
assert call_kwargs["collection"] == "VectorWaveTokenUsage"
|
|
110
|
+
assert props["tokens"] == 45
|
|
111
|
+
assert props["usage_type"] == "embedding"
|
|
112
|
+
assert props["category"] == "test_embed_category"
|
|
@@ -25,46 +25,62 @@ def mock_caching_deps(monkeypatch):
|
|
|
25
25
|
Mocks core dependencies for semantic caching testing.
|
|
26
26
|
Mocking path: vectorwave.utils.return_caching_utils
|
|
27
27
|
"""
|
|
28
|
-
# 1. Mock BatchManager
|
|
28
|
+
# 1. Mock BatchManager
|
|
29
29
|
mock_batch_manager = MagicMock()
|
|
30
30
|
mock_batch_manager.add_object = MagicMock()
|
|
31
31
|
mock_get_batch_manager = MagicMock(return_value=mock_batch_manager)
|
|
32
32
|
|
|
33
|
-
# 2. Mock Vectorizer
|
|
33
|
+
# 2. Mock Vectorizer
|
|
34
34
|
mock_vectorizer = MagicMock()
|
|
35
|
-
mock_vectorizer.embed.return_value = [0.1, 0.2, 0.3]
|
|
35
|
+
mock_vectorizer.embed.return_value = [0.1, 0.2, 0.3]
|
|
36
36
|
mock_get_vectorizer = MagicMock(return_value=mock_vectorizer)
|
|
37
37
|
|
|
38
38
|
# 3. Mock Settings
|
|
39
39
|
mock_settings = WeaviateSettings(
|
|
40
40
|
COLLECTION_NAME="TestFunctions",
|
|
41
41
|
EXECUTION_COLLECTION_NAME="TestExecutions",
|
|
42
|
+
GOLDEN_COLLECTION_NAME="TestGolden", # Ensure this is set
|
|
42
43
|
global_custom_values={"run_id": "test-run-cache"},
|
|
43
|
-
sensitive_keys={"secret_key"}
|
|
44
|
+
sensitive_keys={"secret_key"}
|
|
44
45
|
)
|
|
45
46
|
mock_get_settings = MagicMock(return_value=mock_settings)
|
|
46
47
|
|
|
47
|
-
# 4. Mock Weaviate Client (
|
|
48
|
+
# 4. Mock Weaviate Client (Crucial Fix)
|
|
48
49
|
mock_client = MagicMock()
|
|
50
|
+
|
|
51
|
+
# Setup mock for Golden Dataset query chain:
|
|
52
|
+
# client.collections.get().query.near_vector() -> returns empty objects list by default
|
|
53
|
+
mock_collection = MagicMock()
|
|
54
|
+
mock_query = MagicMock()
|
|
55
|
+
mock_response = MagicMock()
|
|
56
|
+
mock_response.objects = [] # Default to no golden hit
|
|
57
|
+
|
|
58
|
+
mock_query.near_vector.return_value = mock_response
|
|
59
|
+
mock_collection.query = mock_query
|
|
60
|
+
mock_client.collections.get.return_value = mock_collection
|
|
61
|
+
|
|
49
62
|
mock_get_client = MagicMock(return_value=mock_client)
|
|
50
63
|
|
|
51
|
-
# 5. Mock DB Search (
|
|
52
|
-
mock_search_similar_execution = MagicMock(return_value=None)
|
|
64
|
+
# 5. Mock DB Search (Standard Cache)
|
|
65
|
+
mock_search_similar_execution = MagicMock(return_value=None)
|
|
53
66
|
|
|
54
67
|
# --- Apply Mocking ---
|
|
55
68
|
|
|
56
|
-
# Core/Decorator Dependencies
|
|
57
|
-
monkeypatch.setattr("vectorwave.core.decorator.get_batch_manager", mock_get_batch_manager)
|
|
58
|
-
monkeypatch.setattr("vectorwave.core.decorator.get_weaviate_settings", mock_get_settings)
|
|
59
|
-
monkeypatch.setattr("vectorwave.core.decorator.get_vectorizer", mock_get_vectorizer)
|
|
60
|
-
|
|
61
|
-
# Caching Utils Dependencies (Using the correct path)
|
|
62
69
|
MOCK_PATH = "vectorwave.utils.return_caching_utils"
|
|
70
|
+
|
|
71
|
+
# [FIX] Mock get_cached_client in return_caching_utils
|
|
72
|
+
monkeypatch.setattr(f"{MOCK_PATH}.get_cached_client", mock_get_client)
|
|
73
|
+
|
|
63
74
|
monkeypatch.setattr(f"{MOCK_PATH}.get_weaviate_settings", mock_get_settings)
|
|
64
75
|
monkeypatch.setattr(f"{MOCK_PATH}.get_vectorizer", mock_get_vectorizer)
|
|
65
76
|
monkeypatch.setattr(f"{MOCK_PATH}.search_similar_execution", mock_search_similar_execution)
|
|
66
77
|
|
|
67
|
-
#
|
|
78
|
+
# Core/Decorator Dependencies
|
|
79
|
+
monkeypatch.setattr("vectorwave.core.decorator.get_batch_manager", mock_get_batch_manager)
|
|
80
|
+
monkeypatch.setattr("vectorwave.core.decorator.get_weaviate_settings", mock_get_settings)
|
|
81
|
+
monkeypatch.setattr("vectorwave.core.decorator.get_vectorizer", mock_get_vectorizer)
|
|
82
|
+
|
|
83
|
+
# Tracer Dependencies
|
|
68
84
|
monkeypatch.setattr("vectorwave.monitoring.tracer.get_batch_manager", mock_get_batch_manager)
|
|
69
85
|
monkeypatch.setattr("vectorwave.monitoring.tracer.get_weaviate_settings", mock_get_settings)
|
|
70
86
|
monkeypatch.setattr("vectorwave.monitoring.tracer.get_vectorizer", mock_get_vectorizer)
|
|
@@ -79,12 +95,12 @@ def mock_caching_deps(monkeypatch):
|
|
|
79
95
|
real_get_settings.cache_clear()
|
|
80
96
|
real_get_vectorizer.cache_clear()
|
|
81
97
|
|
|
82
|
-
|
|
83
98
|
return {
|
|
84
99
|
"batch": mock_batch_manager,
|
|
85
100
|
"vectorizer": mock_vectorizer,
|
|
86
101
|
"search_cache": mock_search_similar_execution,
|
|
87
|
-
"settings": mock_settings
|
|
102
|
+
"settings": mock_settings,
|
|
103
|
+
"client": mock_client # Return client if you need to manipulate golden cache results in tests
|
|
88
104
|
}
|
|
89
105
|
|
|
90
106
|
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from unittest.mock import MagicMock, patch
|
|
3
|
+
import math
|
|
4
|
+
from vectorwave.database.dataset import VectorWaveDatasetManager
|
|
5
|
+
from vectorwave.models.db_config import WeaviateSettings
|
|
6
|
+
|
|
7
|
+
@pytest.fixture
|
|
8
|
+
def mock_dataset_deps(monkeypatch):
|
|
9
|
+
"""Mocking dependencies for DatasetManager tests"""
|
|
10
|
+
# 1. Settings Mock
|
|
11
|
+
mock_settings = WeaviateSettings(
|
|
12
|
+
EXECUTION_COLLECTION_NAME="Executions",
|
|
13
|
+
GOLDEN_COLLECTION_NAME="GoldenData",
|
|
14
|
+
RECOMMENDATION_STEADY_MARGIN=0.1,
|
|
15
|
+
RECOMMENDATION_DISCOVERY_MARGIN=0.2
|
|
16
|
+
)
|
|
17
|
+
mock_get_settings = MagicMock(return_value=mock_settings)
|
|
18
|
+
|
|
19
|
+
# 2. Client & Collections Mock
|
|
20
|
+
mock_client = MagicMock()
|
|
21
|
+
mock_exec_col = MagicMock()
|
|
22
|
+
mock_golden_col = MagicMock()
|
|
23
|
+
|
|
24
|
+
def get_collection_side_effect(name):
|
|
25
|
+
if name == "Executions": return mock_exec_col
|
|
26
|
+
if name == "GoldenData": return mock_golden_col
|
|
27
|
+
return MagicMock()
|
|
28
|
+
|
|
29
|
+
mock_client.collections.get.side_effect = get_collection_side_effect
|
|
30
|
+
mock_get_client = MagicMock(return_value=mock_client)
|
|
31
|
+
|
|
32
|
+
# 3. Patching
|
|
33
|
+
monkeypatch.setattr("vectorwave.database.dataset.get_cached_client", mock_get_client)
|
|
34
|
+
monkeypatch.setattr("vectorwave.database.dataset.get_weaviate_settings", mock_get_settings)
|
|
35
|
+
|
|
36
|
+
return {
|
|
37
|
+
"exec_col": mock_exec_col,
|
|
38
|
+
"golden_col": mock_golden_col,
|
|
39
|
+
"settings": mock_settings
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
def create_mock_obj(uuid_str, props=None, vector=None):
|
|
43
|
+
obj = MagicMock()
|
|
44
|
+
obj.uuid = uuid_str
|
|
45
|
+
obj.properties = props or {}
|
|
46
|
+
if vector:
|
|
47
|
+
obj.vector = {"default": vector}
|
|
48
|
+
return obj
|
|
49
|
+
|
|
50
|
+
def test_register_as_golden_success(mock_dataset_deps):
|
|
51
|
+
"""[Case 1] Test successful Golden Data registration"""
|
|
52
|
+
manager = VectorWaveDatasetManager()
|
|
53
|
+
|
|
54
|
+
# Arrange: Simulate successful retrieval of original log
|
|
55
|
+
mock_log = create_mock_obj("log-uuid-1", {"function_name": "test_func", "return_value": "res"}, vector=[0.1, 0.2])
|
|
56
|
+
mock_dataset_deps["exec_col"].query.fetch_object_by_id.return_value = mock_log
|
|
57
|
+
|
|
58
|
+
# Act
|
|
59
|
+
result = manager.register_as_golden("log-uuid-1", note="Best case")
|
|
60
|
+
|
|
61
|
+
# Assert
|
|
62
|
+
assert result is True
|
|
63
|
+
# Verify if insert was called on Golden Collection
|
|
64
|
+
mock_dataset_deps["golden_col"].data.insert.assert_called_once()
|
|
65
|
+
call_kwargs = mock_dataset_deps["golden_col"].data.insert.call_args.kwargs
|
|
66
|
+
assert call_kwargs["properties"]["original_uuid"] == "log-uuid-1"
|
|
67
|
+
assert call_kwargs["properties"]["note"] == "Best case"
|
|
68
|
+
assert call_kwargs["vector"] == [0.1, 0.2] # Check if vector was copied
|
|
69
|
+
|
|
70
|
+
def test_recommend_candidates_logic(mock_dataset_deps):
|
|
71
|
+
"""[Case 2] Test density-based recommendation logic (Steady/Discovery)"""
|
|
72
|
+
manager = VectorWaveDatasetManager()
|
|
73
|
+
|
|
74
|
+
# Arrange 1: Golden Data (Set reference point)
|
|
75
|
+
# Centroid: [1.0, 1.0], Avg Dist: 0.0 (Assuming all points are identical)
|
|
76
|
+
golden_vec = [1.0, 1.0]
|
|
77
|
+
golden_objs = [create_mock_obj("gold-1", {"original_uuid": "origin-1"}, golden_vec)]
|
|
78
|
+
mock_dataset_deps["golden_col"].query.fetch_objects.return_value.objects = golden_objs
|
|
79
|
+
|
|
80
|
+
# Cand A: [1.05, 1.05] -> Dist ≈ 0.07 (Steady Range: <= 0.1)
|
|
81
|
+
# Cand B: [1.2, 1.2] -> Dist ≈ 0.28 (Discovery Range: 0.1 < d <= 0.3)
|
|
82
|
+
# Cand C: [2.0, 2.0] -> Dist ≈ 1.41 (Ignore Range: > 0.3)
|
|
83
|
+
cand_a = create_mock_obj("cand-a", {"return_value": "A"}, [1.05, 1.05])
|
|
84
|
+
cand_b = create_mock_obj("cand-b", {"return_value": "B"}, [1.2, 1.2])
|
|
85
|
+
cand_c = create_mock_obj("cand-c", {"return_value": "C"}, [2.0, 2.0])
|
|
86
|
+
|
|
87
|
+
mock_dataset_deps["exec_col"].query.near_vector.return_value.objects = [cand_a, cand_b, cand_c]
|
|
88
|
+
|
|
89
|
+
# Act
|
|
90
|
+
recommendations = manager.recommend_candidates("test_func")
|
|
91
|
+
|
|
92
|
+
# Assert
|
|
93
|
+
assert len(recommendations) == 2
|
|
94
|
+
assert recommendations[0]["uuid"] == "cand-a"
|
|
95
|
+
assert recommendations[0]["type"] == "STEADY"
|
|
96
|
+
assert recommendations[1]["uuid"] == "cand-b"
|
|
97
|
+
assert recommendations[1]["type"] == "DISCOVERY"
|
|
@@ -4,17 +4,19 @@ import asyncio
|
|
|
4
4
|
import inspect
|
|
5
5
|
from unittest.mock import MagicMock, patch
|
|
6
6
|
from vectorwave.utils.replayer import VectorWaveReplayer
|
|
7
|
+
from vectorwave.models.db_config import WeaviateSettings
|
|
7
8
|
|
|
8
9
|
# --- 1. Mock Fixtures (Mock Environment Setup) ---
|
|
9
10
|
|
|
10
11
|
@pytest.fixture
|
|
11
12
|
def mock_replayer_deps(monkeypatch):
|
|
12
13
|
"""
|
|
13
|
-
Mocks the DB client and settings used by the Replayer.
|
|
14
|
+
Mocks the DB client and settings used by the Replayer (Default Setup).
|
|
14
15
|
"""
|
|
15
16
|
# Settings Mock
|
|
16
17
|
mock_settings = MagicMock()
|
|
17
18
|
mock_settings.EXECUTION_COLLECTION_NAME = "VectorWaveExecutions"
|
|
19
|
+
mock_settings.GOLDEN_COLLECTION_NAME = "VectorWaveGoldenDataset"
|
|
18
20
|
|
|
19
21
|
# Weaviate Client & Collection Mock
|
|
20
22
|
mock_client = MagicMock()
|
|
@@ -26,7 +28,7 @@ def mock_replayer_deps(monkeypatch):
|
|
|
26
28
|
mock_query.fetch_objects.return_value = MagicMock(objects=[])
|
|
27
29
|
mock_collection.query = mock_query
|
|
28
30
|
|
|
29
|
-
# Data Operation Mock
|
|
31
|
+
# Data Operation Mock
|
|
30
32
|
mock_data = MagicMock()
|
|
31
33
|
mock_collection.data = mock_data
|
|
32
34
|
|
|
@@ -40,39 +42,58 @@ def mock_replayer_deps(monkeypatch):
|
|
|
40
42
|
"data": mock_data
|
|
41
43
|
}
|
|
42
44
|
|
|
45
|
+
@pytest.fixture
|
|
46
|
+
def mock_replayer_deps_v2(monkeypatch):
|
|
47
|
+
"""
|
|
48
|
+
Mock fixture separating Golden and Execution collections for Priority Testing.
|
|
49
|
+
"""
|
|
50
|
+
# Settings
|
|
51
|
+
mock_settings = WeaviateSettings(
|
|
52
|
+
EXECUTION_COLLECTION_NAME="Executions",
|
|
53
|
+
GOLDEN_COLLECTION_NAME="GoldenData"
|
|
54
|
+
)
|
|
55
|
+
mock_get_settings = MagicMock(return_value=mock_settings)
|
|
56
|
+
|
|
57
|
+
# Client & Collections
|
|
58
|
+
mock_client = MagicMock()
|
|
59
|
+
mock_exec_col = MagicMock()
|
|
60
|
+
mock_golden_col = MagicMock()
|
|
61
|
+
|
|
62
|
+
def get_collection_side_effect(name):
|
|
63
|
+
if name == "Executions": return mock_exec_col
|
|
64
|
+
if name == "GoldenData": return mock_golden_col
|
|
65
|
+
return MagicMock()
|
|
66
|
+
|
|
67
|
+
mock_client.collections.get.side_effect = get_collection_side_effect
|
|
68
|
+
mock_get_client = MagicMock(return_value=mock_client)
|
|
69
|
+
|
|
70
|
+
monkeypatch.setattr("vectorwave.utils.replayer.get_cached_client", mock_get_client)
|
|
71
|
+
monkeypatch.setattr("vectorwave.utils.replayer.get_weaviate_settings", mock_get_settings)
|
|
72
|
+
|
|
73
|
+
return {
|
|
74
|
+
"golden_col": mock_golden_col,
|
|
75
|
+
"exec_col": mock_exec_col
|
|
76
|
+
}
|
|
77
|
+
|
|
43
78
|
def create_mock_log(uuid_str, inputs, return_value):
|
|
44
79
|
"""Mimics a log object retrieved from the database."""
|
|
45
80
|
mock_obj = MagicMock()
|
|
46
81
|
mock_obj.uuid = uuid_str
|
|
47
|
-
|
|
48
|
-
# Combine inputs and return_value into properties
|
|
49
82
|
props = inputs.copy()
|
|
50
83
|
props["return_value"] = json.dumps(return_value) if not isinstance(return_value, str) else return_value
|
|
51
84
|
props["timestamp_utc"] = "2023-01-01T00:00:00Z"
|
|
52
|
-
|
|
53
85
|
mock_obj.properties = props
|
|
54
86
|
return mock_obj
|
|
55
87
|
|
|
56
88
|
# --- 2. Test Cases ---
|
|
57
89
|
|
|
58
90
|
def test_replay_success_match(mock_replayer_deps):
|
|
59
|
-
"""
|
|
60
|
-
[Case 1] Successful Pass: Checks if the DB value matches the current function execution result.
|
|
61
|
-
"""
|
|
62
|
-
# Arrange
|
|
91
|
+
"""[Case 1] Successful Pass"""
|
|
63
92
|
replayer = VectorWaveReplayer()
|
|
64
|
-
|
|
65
|
-
# 1. DB Mock Data (Input: a=1, b=2 -> Expected: 3)
|
|
66
93
|
mock_logs = [create_mock_log("uuid-1", {"a": 1, "b": 2}, 3)]
|
|
67
94
|
mock_replayer_deps["query"].fetch_objects.return_value.objects = mock_logs
|
|
68
95
|
|
|
69
|
-
|
|
70
|
-
# If 'my_module.add' is called, this lambda function is executed
|
|
71
|
-
mock_func = MagicMock(return_value=3) # Actual result is also 3
|
|
72
|
-
|
|
73
|
-
# Set signature for replayer's inspect.signature check
|
|
74
|
-
# (Mock objects normally lack signatures, so we overwrite it)
|
|
75
|
-
import inspect
|
|
96
|
+
mock_func = MagicMock(return_value=3)
|
|
76
97
|
mock_func.__signature__ = inspect.Signature([
|
|
77
98
|
inspect.Parameter('a', inspect.Parameter.POSITIONAL_OR_KEYWORD),
|
|
78
99
|
inspect.Parameter('b', inspect.Parameter.POSITIONAL_OR_KEYWORD)
|
|
@@ -83,30 +104,19 @@ def test_replay_success_match(mock_replayer_deps):
|
|
|
83
104
|
setattr(mock_module, "add", mock_func)
|
|
84
105
|
mock_import.return_value = mock_module
|
|
85
106
|
|
|
86
|
-
# Act
|
|
87
107
|
result = replayer.replay("my_module.add", limit=1)
|
|
88
108
|
|
|
89
|
-
# Assert
|
|
90
109
|
assert result["passed"] == 1
|
|
91
110
|
assert result["failed"] == 0
|
|
92
|
-
mock_func.assert_called_with(a=1, b=2)
|
|
93
|
-
|
|
111
|
+
mock_func.assert_called_with(a=1, b=2)
|
|
94
112
|
|
|
95
113
|
def test_replay_failure_mismatch(mock_replayer_deps):
|
|
96
|
-
"""
|
|
97
|
-
[Case 2] Failure: Checks for mismatch when the result value is different (Regression).
|
|
98
|
-
"""
|
|
99
|
-
# Arrange
|
|
114
|
+
"""[Case 2] Failure: Regression check"""
|
|
100
115
|
replayer = VectorWaveReplayer()
|
|
101
|
-
|
|
102
|
-
# DB: 1 + 2 = 3
|
|
103
116
|
mock_logs = [create_mock_log("uuid-2", {"a": 1, "b": 2}, 3)]
|
|
104
117
|
mock_replayer_deps["query"].fetch_objects.return_value.objects = mock_logs
|
|
105
118
|
|
|
106
|
-
|
|
107
|
-
mock_func = MagicMock(return_value=99)
|
|
108
|
-
|
|
109
|
-
import inspect
|
|
119
|
+
mock_func = MagicMock(return_value=99) # Bug
|
|
110
120
|
mock_func.__signature__ = inspect.Signature([
|
|
111
121
|
inspect.Parameter('a', inspect.Parameter.POSITIONAL_OR_KEYWORD),
|
|
112
122
|
inspect.Parameter('b', inspect.Parameter.POSITIONAL_OR_KEYWORD)
|
|
@@ -116,33 +126,20 @@ def test_replay_failure_mismatch(mock_replayer_deps):
|
|
|
116
126
|
mock_module = MagicMock()
|
|
117
127
|
setattr(mock_module, "add", mock_func)
|
|
118
128
|
mock_import.return_value = mock_module
|
|
119
|
-
|
|
120
|
-
# Act
|
|
121
129
|
result = replayer.replay("my_module.add")
|
|
122
130
|
|
|
123
|
-
# Assert
|
|
124
131
|
assert result["passed"] == 0
|
|
125
132
|
assert result["failed"] == 1
|
|
126
|
-
assert len(result["failures"]) == 1
|
|
127
133
|
assert result["failures"][0]["expected"] == 3
|
|
128
134
|
assert result["failures"][0]["actual"] == 99
|
|
129
135
|
|
|
130
|
-
|
|
131
136
|
def test_replay_update_baseline(mock_replayer_deps):
|
|
132
|
-
"""
|
|
133
|
-
[Case 3] Update: Checks updating the baseline when the result is different but update_baseline=True.
|
|
134
|
-
"""
|
|
135
|
-
# Arrange
|
|
137
|
+
"""[Case 3] Update Baseline"""
|
|
136
138
|
replayer = VectorWaveReplayer()
|
|
137
|
-
|
|
138
|
-
# DB: Old value 'Old'
|
|
139
139
|
mock_logs = [create_mock_log("uuid-3", {"msg": "Hi"}, "Old")]
|
|
140
140
|
mock_replayer_deps["query"].fetch_objects.return_value.objects = mock_logs
|
|
141
141
|
|
|
142
|
-
# Func: New value 'New'
|
|
143
142
|
mock_func = MagicMock(return_value="New")
|
|
144
|
-
|
|
145
|
-
import inspect
|
|
146
143
|
mock_func.__signature__ = inspect.Signature([
|
|
147
144
|
inspect.Parameter('msg', inspect.Parameter.POSITIONAL_OR_KEYWORD)
|
|
148
145
|
])
|
|
@@ -151,37 +148,22 @@ def test_replay_update_baseline(mock_replayer_deps):
|
|
|
151
148
|
mock_module = MagicMock()
|
|
152
149
|
setattr(mock_module, "greet", mock_func)
|
|
153
150
|
mock_import.return_value = mock_module
|
|
154
|
-
|
|
155
|
-
# Act
|
|
156
|
-
# Set update_baseline=True
|
|
157
151
|
result = replayer.replay("my_module.greet", update_baseline=True)
|
|
158
152
|
|
|
159
|
-
# Assert
|
|
160
153
|
assert result["updated"] == 1
|
|
161
|
-
# Verify the DB update function was called
|
|
162
154
|
mock_replayer_deps["data"].update.assert_called_once_with(
|
|
163
155
|
uuid="uuid-3",
|
|
164
|
-
properties={"return_value": '"New"'}
|
|
156
|
+
properties={"return_value": '"New"'}
|
|
165
157
|
)
|
|
166
158
|
|
|
167
|
-
|
|
168
159
|
def test_replay_argument_filtering(mock_replayer_deps):
|
|
169
|
-
"""
|
|
170
|
-
[Case 4] Argument Filtering: Checks that unnecessary metadata (like user_id) not in the function signature is removed.
|
|
171
|
-
"""
|
|
172
|
-
# Arrange
|
|
160
|
+
"""[Case 4] Argument Filtering"""
|
|
173
161
|
replayer = VectorWaveReplayer()
|
|
174
|
-
|
|
175
|
-
# DB contains extraneous data like 'team', 'timestamp', etc.
|
|
176
162
|
inputs = {"a": 10, "team": "billing", "priority": 1}
|
|
177
163
|
mock_logs = [create_mock_log("uuid-4", inputs, 100)]
|
|
178
164
|
mock_replayer_deps["query"].fetch_objects.return_value.objects = mock_logs
|
|
179
165
|
|
|
180
|
-
# Function only accepts 'a' as an argument
|
|
181
166
|
mock_func = MagicMock(return_value=100)
|
|
182
|
-
|
|
183
|
-
import inspect
|
|
184
|
-
# Only 'a' is defined in the signature
|
|
185
167
|
mock_func.__signature__ = inspect.Signature([
|
|
186
168
|
inspect.Parameter('a', inspect.Parameter.POSITIONAL_OR_KEYWORD)
|
|
187
169
|
])
|
|
@@ -190,52 +172,76 @@ def test_replay_argument_filtering(mock_replayer_deps):
|
|
|
190
172
|
mock_module = MagicMock()
|
|
191
173
|
setattr(mock_module, "calc", mock_func)
|
|
192
174
|
mock_import.return_value = mock_module
|
|
193
|
-
|
|
194
|
-
# Act
|
|
195
175
|
replayer.replay("my_module.calc")
|
|
196
176
|
|
|
197
|
-
# Assert
|
|
198
|
-
# Should only be called with 'a=10', excluding 'team' and 'priority'
|
|
199
177
|
mock_func.assert_called_once_with(a=10)
|
|
200
178
|
|
|
201
179
|
def test_replay_async_function_execution_fixed(mock_replayer_deps):
|
|
202
|
-
"""
|
|
203
|
-
[Case 5] Async Function Test (FIXED): Tests the async execution path using patching
|
|
204
|
-
of the import mechanism and executing the actual async function via asyncio.run.
|
|
205
|
-
"""
|
|
206
|
-
# Arrange
|
|
180
|
+
"""[Case 5] Async Function Test"""
|
|
207
181
|
replayer = VectorWaveReplayer()
|
|
208
|
-
|
|
209
|
-
# 1. DB Mock Data
|
|
210
182
|
inputs = {"a": 1, "b": 2}
|
|
211
183
|
expected_result = 3
|
|
212
184
|
mock_logs = [create_mock_log("uuid-async-1", inputs, expected_result)]
|
|
213
185
|
mock_replayer_deps["query"].fetch_objects.return_value.objects = mock_logs
|
|
214
186
|
|
|
215
|
-
# 2. Define the actual ASYNC function for replayer to execute
|
|
216
187
|
async def real_async_add(a, b):
|
|
217
|
-
# This function will be called and executed by asyncio.run
|
|
218
188
|
await asyncio.sleep(0.001)
|
|
219
189
|
return a + b
|
|
220
190
|
|
|
221
|
-
# Manually attach the signature for the replayer's inspection check to pass
|
|
222
191
|
setattr(real_async_add, '__signature__', inspect.Signature([
|
|
223
192
|
inspect.Parameter('a', inspect.Parameter.POSITIONAL_OR_KEYWORD),
|
|
224
193
|
inspect.Parameter('b', inspect.Parameter.POSITIONAL_OR_KEYWORD)
|
|
225
194
|
]))
|
|
226
195
|
|
|
227
|
-
# 3. Patch importlib.import_module to return a mock module that contains the target function
|
|
228
196
|
mock_module = MagicMock()
|
|
229
|
-
mock_module.async_add = real_async_add
|
|
197
|
+
mock_module.async_add = real_async_add
|
|
230
198
|
|
|
231
199
|
with patch("vectorwave.utils.replayer.importlib.import_module", return_value=mock_module):
|
|
232
|
-
|
|
233
|
-
# Act
|
|
234
|
-
# replayer.replay (sync function) calls asyncio.run(real_async_add(**inputs)) internally.
|
|
235
200
|
result = replayer.replay("my_module.async_add", limit=1)
|
|
236
201
|
|
|
237
|
-
# Assert
|
|
238
|
-
# 1. The result should be successful and match the expected output
|
|
239
202
|
assert result["passed"] == 1
|
|
240
203
|
assert result["failed"] == 0
|
|
241
|
-
|
|
204
|
+
|
|
205
|
+
def test_replay_fetches_golden_first(mock_replayer_deps_v2):
|
|
206
|
+
"""
|
|
207
|
+
[Case 6] Test if Replayer prioritizes fetching Golden Data
|
|
208
|
+
"""
|
|
209
|
+
from vectorwave.utils.replayer import VectorWaveReplayer
|
|
210
|
+
|
|
211
|
+
# Arrange
|
|
212
|
+
# 1. Setup one Golden Data entry
|
|
213
|
+
golden_obj = MagicMock()
|
|
214
|
+
golden_obj.uuid = "golden-uuid"
|
|
215
|
+
golden_obj.properties = {"original_uuid": "orig-1", "return_value": "3"}
|
|
216
|
+
mock_replayer_deps_v2["golden_col"].query.fetch_objects.return_value.objects = [golden_obj]
|
|
217
|
+
|
|
218
|
+
# Retrieve original log (to get input values)
|
|
219
|
+
orig_log = MagicMock()
|
|
220
|
+
orig_log.properties = {"a": 1, "b": 2}
|
|
221
|
+
mock_replayer_deps_v2["exec_col"].query.fetch_object_by_id.return_value = orig_log
|
|
222
|
+
|
|
223
|
+
# 2. Leave Standard Data empty
|
|
224
|
+
mock_replayer_deps_v2["exec_col"].query.fetch_objects.return_value.objects = []
|
|
225
|
+
|
|
226
|
+
# Function Mock
|
|
227
|
+
mock_func = MagicMock(return_value=3)
|
|
228
|
+
mock_func.__signature__ = inspect.Signature([
|
|
229
|
+
inspect.Parameter('a', inspect.Parameter.POSITIONAL_OR_KEYWORD),
|
|
230
|
+
inspect.Parameter('b', inspect.Parameter.POSITIONAL_OR_KEYWORD)
|
|
231
|
+
])
|
|
232
|
+
|
|
233
|
+
# Act
|
|
234
|
+
replayer = VectorWaveReplayer()
|
|
235
|
+
with patch("vectorwave.utils.replayer.importlib.import_module") as mock_import:
|
|
236
|
+
mock_module = MagicMock()
|
|
237
|
+
setattr(mock_module, "add", mock_func)
|
|
238
|
+
mock_import.return_value = mock_module
|
|
239
|
+
|
|
240
|
+
result = replayer.replay("mod.add", limit=10)
|
|
241
|
+
|
|
242
|
+
# Assert
|
|
243
|
+
assert result["total"] == 1
|
|
244
|
+
# Verify that the Golden Collection was queried
|
|
245
|
+
mock_replayer_deps_v2["golden_col"].query.fetch_objects.assert_called_once()
|
|
246
|
+
# Verify that fetch_object_by_id was called to retrieve the original log
|
|
247
|
+
mock_replayer_deps_v2["exec_col"].query.fetch_object_by_id.assert_called_with("orig-1")
|