matrixone-python-sdk 0.1.0__py3-none-any.whl

tests/offline/test_pinecone_filter_offline.py

@@ -0,0 +1,377 @@
+# Copyright 2021 - 2022 Matrix Origin
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Offline tests for Pinecone-compatible filter functionality.
+These tests focus on logic validation, parameter handling, and result structure
+without requiring actual database connections.
+"""
+
+import pytest
+from unittest.mock import Mock, MagicMock, patch
+from typing import List, Dict, Any
+from matrixone.search_vector_index import PineconeCompatibleIndex, QueryResponse, VectorMatch
+
+
+class TestPineconeFilterOffline:
+    """Offline tests for Pinecone-compatible filter functionality"""
+
+    @pytest.fixture
+    def mock_client(self):
+        """Create mock client for testing"""
+        client = Mock()
+        client.execute = Mock()
+        client.vector_query = Mock()
+        client.vector_query.similarity_search = Mock()
+        return client
+
+    @pytest.fixture
+    def mock_index(self, mock_client):
+        """Create mock Pinecone-compatible index"""
+        index = PineconeCompatibleIndex(mock_client, "test_table", "embedding")
+        return index
+
+    def test_filter_parsing_empty_dict(self, mock_index):
+        """Test filter parsing with empty dictionary"""
+        where_conditions, where_params = mock_index._parse_pinecone_filter({})
+        assert where_conditions == []
+        assert where_params == []
+
+    def test_filter_parsing_none_value(self, mock_index):
+        """Test filter parsing with None value"""
+        where_conditions, where_params = mock_index._parse_pinecone_filter(None)
+        assert where_conditions == []
+        assert where_params == []
+
+    def test_filter_parsing_basic_equality(self, mock_index):
+        """Test filter parsing with basic equality"""
+        filter_dict = {"genre": "action"}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["genre = ?"]
+        assert where_params == ["action"]
+
+    def test_filter_parsing_operators(self, mock_index):
+        """Test filter parsing with various operators"""
+        # Test $eq
+        filter_dict = {"rating": {"$eq": 8.5}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["rating = ?"]
+        assert where_params == [8.5]
+
+        # Test $ne
+        filter_dict = {"genre": {"$ne": "horror"}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["genre != ?"]
+        assert where_params == ["horror"]
+
+        # Test $gt
+        filter_dict = {"year": {"$gt": 2000}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["year > ?"]
+        assert where_params == [2000]
+
+        # Test $gte
+        filter_dict = {"rating": {"$gte": 8.0}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["rating >= ?"]
+        assert where_params == [8.0]
+
+        # Test $lt
+        filter_dict = {"year": {"$lt": 2010}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["year < ?"]
+        assert where_params == [2010]
+
+        # Test $lte
+        filter_dict = {"rating": {"$lte": 9.0}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["rating <= ?"]
+        assert where_params == [9.0]
+
+    def test_filter_parsing_in_operator(self, mock_index):
+        """Test filter parsing with $in operator"""
+        filter_dict = {"genre": {"$in": ["action", "sci-fi", "drama"]}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["genre IN (?,?,?)"]
+        assert where_params == ["action", "sci-fi", "drama"]
+
+    def test_filter_parsing_nin_operator(self, mock_index):
+        """Test filter parsing with $nin operator"""
+        filter_dict = {"genre": {"$nin": ["horror", "thriller"]}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["genre NOT IN (?,?)"]
+        assert where_params == ["horror", "thriller"]
+
+    def test_filter_parsing_empty_in_list(self, mock_index):
+        """Test filter parsing with empty $in list"""
+        filter_dict = {"genre": {"$in": []}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["1=0"]  # Always false condition
+        assert where_params == []
+
+    def test_filter_parsing_empty_nin_list(self, mock_index):
+        """Test filter parsing with empty $nin list"""
+        filter_dict = {"genre": {"$nin": []}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["1=1"]  # Always true condition
+        assert where_params == []
+
+    def test_filter_parsing_and_operator(self, mock_index):
+        """Test filter parsing with $and operator"""
+        filter_dict = {"$and": [{"genre": "action"}, {"year": {"$gte": 2000}}]}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["(genre = ? AND year >= ?)"]
+        assert where_params == ["action", 2000]
+
+    def test_filter_parsing_or_operator(self, mock_index):
+        """Test filter parsing with $or operator"""
+        filter_dict = {"$or": [{"genre": "action"}, {"genre": "sci-fi"}]}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["(genre = ? OR genre = ?)"]
+        assert where_params == ["action", "sci-fi"]
+
+    def test_filter_parsing_nested_conditions(self, mock_index):
+        """Test filter parsing with nested $and and $or conditions"""
+        filter_dict = {
+            "$and": [
+                {"$or": [{"genre": "action"}, {"genre": "sci-fi"}]},
+                {"$and": [{"year": {"$gte": 2000}}, {"rating": {"$gte": 8.0}}]},
+            ]
+        }
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        expected_condition = "((genre = ? OR genre = ?) AND (year >= ? AND rating >= ?))"
+        assert where_conditions == [expected_condition]
+        assert where_params == ["action", "sci-fi", 2000, 8.0]
+
+    def test_filter_parsing_special_characters(self, mock_index):
+        """Test filter parsing with special characters"""
+        filter_dict = {"director": "Director with apostrophe's name"}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["director = ?"]
+        assert where_params == ["Director with apostrophe's name"]
+
+    def test_filter_parsing_mixed_data_types(self, mock_index):
+        """Test filter parsing with mixed data types"""
+        filter_dict = {"year": {"$in": [1999, "2010", 2008]}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["year IN (?,?,?)"]
+        assert where_params == [1999, "2010", 2008]
+
+    def test_filter_parsing_large_in_list(self, mock_index):
+        """Test filter parsing with large $in list"""
+        large_list = [str(i) for i in range(1000)]
+        large_list.extend(["action", "sci-fi"])
+        filter_dict = {"genre": {"$in": large_list}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+
+        # Check that placeholders are created correctly
+        expected_placeholders = ",".join(["?" for _ in range(1002)])
+        assert where_conditions == [f"genre IN ({expected_placeholders})"]
+        assert len(where_params) == 1002
+
+    def test_vector_dimension_validation(self, mock_index):
+        """Test vector dimension validation logic"""
+        # Test with valid vector
+        valid_vector = [0.1] * 64
+        # This should not raise an exception during parsing
+        where_conditions, where_params = mock_index._parse_pinecone_filter({"genre": "action"})
+        assert where_conditions == ["genre = ?"]
+        assert where_params == ["action"]
+
+    def test_top_k_validation_logic(self, mock_index):
+        """Test top_k validation logic"""
+        # Test various top_k values
+        test_cases = [0, 1, 10, 100, 10000]
+        for top_k in test_cases:
+            # The logic should handle any positive integer
+            assert isinstance(top_k, int)
+            assert top_k >= 0
+
+    def test_query_response_structure(self):
+        """Test QueryResponse structure validation"""
+        # Create mock matches
+        matches = [
+            VectorMatch(
+                id="1",
+                score=0.1,
+                metadata={"title": "Movie 1", "genre": "action"},
+                values=[0.1] * 64,
+            ),
+            VectorMatch(
+                id="2",
+                score=0.2,
+                metadata={"title": "Movie 2", "genre": "sci-fi"},
+                values=[0.2] * 64,
+            ),
+        ]
+
+        response = QueryResponse(matches=matches, namespace="test", usage={"read_units": 2})
+
+        # Validate structure
+        assert len(response.matches) == 2
+        assert response.namespace == "test"
+        assert response.usage["read_units"] == 2
+
+        # Validate match structure
+        for match in response.matches:
+            assert hasattr(match, 'id')
+            assert hasattr(match, 'score')
+            assert hasattr(match, 'metadata')
+            assert hasattr(match, 'values')
+            assert isinstance(match.score, (int, float))
+            assert match.score >= 0
+
+    def test_vector_match_structure(self):
+        """Test VectorMatch structure validation"""
+        match = VectorMatch(id="test_id", score=0.5, metadata={"key": "value"}, values=[0.1, 0.2, 0.3])
+
+        assert match.id == "test_id"
+        assert match.score == 0.5
+        assert match.metadata == {"key": "value"}
+        assert match.values == [0.1, 0.2, 0.3]
+
+    def test_result_ordering_logic(self):
+        """Test result ordering logic validation"""
+        # Create matches with different scores
+        matches = [
+            VectorMatch(id="1", score=0.3, metadata={}, values=None),
+            VectorMatch(id="2", score=0.1, metadata={}, values=None),
+            VectorMatch(id="3", score=0.2, metadata={}, values=None),
+        ]
+
+        # Sort by score (ascending for distance)
+        sorted_matches = sorted(matches, key=lambda x: x.score)
+
+        # Validate ordering
+        assert sorted_matches[0].score == 0.1
+        assert sorted_matches[1].score == 0.2
+        assert sorted_matches[2].score == 0.3
+
+    def test_usage_statistics_logic(self):
+        """Test usage statistics logic validation"""
+        # Test various scenarios
+        test_cases = [
+            (0, 0),  # top_k=0, expected_read_units=0
+            (1, 1),  # top_k=1, expected_read_units=1
+            (5, 5),  # top_k=5, expected_read_units=5
+            (100, 5),  # top_k=100, but only 5 records available
+        ]
+
+        for top_k, expected_read_units in test_cases:
+            # Simulate the logic
+            actual_read_units = min(top_k, 5)  # Assuming 5 records available
+            assert actual_read_units == expected_read_units
+
+    def test_namespace_consistency_logic(self):
+        """Test namespace consistency logic validation"""
+        test_namespaces = ["", "default", "test_namespace", "namespace_with_underscores"]
+
+        for namespace in test_namespaces:
+            # Namespace should be preserved as-is
+            assert isinstance(namespace, str)
+            # Should be able to handle any string value
+            processed_namespace = namespace
+            assert processed_namespace == namespace
+
+    def test_include_parameters_logic(self):
+        """Test include_metadata and include_values parameter logic"""
+        # Test all combinations
+        test_cases = [
+            (True, True),  # include both
+            (True, False),  # include metadata only
+            (False, True),  # include values only
+            (False, False),  # include neither
+        ]
+
+        for include_metadata, include_values in test_cases:
+            # Validate parameter types
+            assert isinstance(include_metadata, bool)
+            assert isinstance(include_values, bool)
+
+            # Test logic combinations
+            if include_metadata and include_values:
+                # Both included
+                pass
+            elif include_metadata and not include_values:
+                # Metadata only
+                pass
+            elif not include_metadata and include_values:
+                # Values only
+                pass
+            else:
+                # Neither included
+                pass
+
+    def test_vector_value_handling_logic(self):
+        """Test vector value handling logic"""
+        # Test different vector representations
+        test_vectors = [
+            [0.1] * 64,  # List of floats
+            [0.0] * 64,  # Zero vector
+            [-0.1] * 64,  # Negative values
+            [100.0] * 64,  # Large values
+            [0.1, 0.2, 0.3],  # Short vector (should be handled)
+        ]
+
+        for vector in test_vectors:
+            # Validate vector structure
+            assert isinstance(vector, list)
+            assert all(isinstance(x, (int, float)) for x in vector)
+
+            # Test vector string conversion logic
+            vector_str = "[" + ",".join(map(str, vector)) + "]"
+            assert vector_str.startswith("[")
+            assert vector_str.endswith("]")
+
+    def test_error_handling_logic(self):
+        """Test error handling logic for various edge cases"""
+        # Test unsupported operators
+        with pytest.raises(ValueError, match="Unsupported operator"):
+            mock_index = PineconeCompatibleIndex(Mock(), "test", "embedding")
+            mock_index._parse_pinecone_filter({"field": {"$unsupported": "value"}})
+
+    def test_consistency_logic(self, mock_index):
+        """Test consistency logic for multiple operations"""
+        # Test that same input produces same output
+        filter_dict = {"genre": "action", "year": {"$gte": 2000}}
+
+        # Parse multiple times
+        results = []
+        for _ in range(3):
+            where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+            results.append((where_conditions, where_params))
+
+        # All results should be identical
+        for i in range(1, len(results)):
+            assert results[i] == results[0]
+
+    def test_edge_case_handling(self, mock_index):
+        """Test edge case handling logic"""
+        # Test with very large numbers
+        large_number = 999999999
+        filter_dict = {"id": {"$eq": large_number}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["id = ?"]
+        assert where_params == [large_number]
+
+        # Test with very small numbers
+        small_number = 0.000001
+        filter_dict = {"rating": {"$eq": small_number}}
+        where_conditions, where_params = mock_index._parse_pinecone_filter(filter_dict)
+        assert where_conditions == ["rating = ?"]
+        assert where_params == [small_number]
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])