vector-inspector 0.2.0__py3-none-any.whl

vector_inspector/core/connections/template_connection.py

@@ -0,0 +1,346 @@
+"""Template for implementing a new vector database connection.
+
+Copy this file and replace 'Template' with your database name.
+Implement all abstract methods according to your database's API.
+"""
+
+from typing import Optional, List, Dict, Any
+from .base_connection import VectorDBConnection
+
+
+class TemplateConnection(VectorDBConnection):
+    """Template vector database connection.
+    
+    Replace this with your database provider name (e.g., PineconeConnection, QdrantConnection).
+    """
+    
+    def __init__(self, **kwargs):
+        """
+        Initialize connection parameters.
+        
+        Args:
+            **kwargs: Provider-specific connection parameters
+                      (e.g., api_key, host, port, credentials, etc.)
+        """
+        # Store your connection parameters here
+        self._client = None
+        # Add your provider-specific attributes
+        
+    def connect(self) -> bool:
+        """
+        Establish connection to the vector database.
+        
+        Returns:
+            True if connection successful, False otherwise
+        """
+        try:
+            # Initialize your database client here
+            # self._client = YourDatabaseClient(...)
+            return True
+        except Exception as e:
+            print(f"Connection failed: {e}")
+            return False
+    
+    def disconnect(self):
+        """Close connection to the vector database."""
+        # Clean up your connection
+        self._client = None
+    
+    @property
+    def is_connected(self) -> bool:
+        """
+        Check if connected to the vector database.
+        
+        Returns:
+            True if connected, False otherwise
+        """
+        # Return whether the client is active
+        return self._client is not None
+    
+    def list_collections(self) -> List[str]:
+        """
+        Get list of all collections/indexes.
+        
+        Returns:
+            List of collection/index names
+        """
+        if not self._client:
+            return []
+        try:
+            # Call your database API to list collections
+            # collections = self._client.list_collections()
+            # return [col.name for col in collections]
+            return []
+        except Exception as e:
+            print(f"Failed to list collections: {e}")
+            return []
+    
+    def get_collection_info(self, name: str) -> Optional[Dict[str, Any]]:
+        """
+        Get collection metadata and statistics.
+        
+        Args:
+            name: Collection/index name
+            
+        Returns:
+            Dictionary with collection info:
+                - name: Collection name
+                - count: Number of items
+                - metadata_fields: List of available metadata field names
+        """
+        if not self._client:
+            return None
+        
+        try:
+            # Get collection stats from your database
+            # collection = self._client.get_collection(name)
+            # count = collection.count()
+            # metadata_fields = collection.get_metadata_fields()
+            
+            return {
+                "name": name,
+                "count": 0,  # Replace with actual count
+                "metadata_fields": [],  # Replace with actual fields
+            }
+        except Exception as e:
+            print(f"Failed to get collection info: {e}")
+            return None
+    
+    def query_collection(
+        self,
+        collection_name: str,
+        query_texts: Optional[List[str]] = None,
+        query_embeddings: Optional[List[List[float]]] = None,
+        n_results: int = 10,
+        where: Optional[Dict[str, Any]] = None,
+        where_document: Optional[Dict[str, Any]] = None,
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Query a collection for similar vectors.
+        
+        Args:
+            collection_name: Name of collection to query
+            query_texts: Text queries to embed and search
+            query_embeddings: Direct embedding vectors to search
+            n_results: Number of results to return
+            where: Metadata filter
+            where_document: Document content filter
+            
+        Returns:
+            Query results dictionary with keys:
+                - ids: List of result IDs
+                - distances: List of distances/scores
+                - documents: List of document texts
+                - metadatas: List of metadata dicts
+                - embeddings: List of embedding vectors (optional)
+        """
+        if not self._client:
+            return None
+        
+        try:
+            # Perform similarity search
+            # results = self._client.query(
+            #     collection=collection_name,
+            #     query_embeddings=query_embeddings,
+            #     n_results=n_results,
+            #     filter=where
+            # )
+            
+            # Transform results to standard format
+            return {
+                "ids": [],
+                "distances": [],
+                "documents": [],
+                "metadatas": [],
+                "embeddings": []
+            }
+        except Exception as e:
+            print(f"Query failed: {e}")
+            return None
+    
+    def get_all_items(
+        self,
+        collection_name: str,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        where: Optional[Dict[str, Any]] = None,
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Get all items from a collection.
+        
+        Args:
+            collection_name: Name of collection
+            limit: Maximum number of items to return
+            offset: Number of items to skip
+            where: Metadata filter
+            
+        Returns:
+            Dictionary with collection items:
+                - ids: List of item IDs
+                - documents: List of document texts
+                - metadatas: List of metadata dicts
+                - embeddings: List of embedding vectors
+        """
+        if not self._client:
+            return None
+        
+        try:
+            # Fetch items from collection with pagination
+            # results = self._client.fetch(
+            #     collection=collection_name,
+            #     limit=limit,
+            #     offset=offset,
+            #     filter=where
+            # )
+            
+            return {
+                "ids": [],
+                "documents": [],
+                "metadatas": [],
+                "embeddings": []
+            }
+        except Exception as e:
+            print(f"Failed to get items: {e}")
+            return None
+    
+    def add_items(
+        self,
+        collection_name: str,
+        documents: List[str],
+        metadatas: Optional[List[Dict[str, Any]]] = None,
+        ids: Optional[List[str]] = None,
+        embeddings: Optional[List[List[float]]] = None,
+    ) -> bool:
+        """
+        Add items to a collection.
+        
+        Args:
+            collection_name: Name of collection
+            documents: Document texts
+            metadatas: Metadata for each document
+            ids: IDs for each document
+            embeddings: Pre-computed embeddings
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        if not self._client:
+            return False
+        
+        try:
+            # Add items to the collection
+            # self._client.upsert(
+            #     collection=collection_name,
+            #     documents=documents,
+            #     metadatas=metadatas,
+            #     ids=ids,
+            #     embeddings=embeddings
+            # )
+            return True
+        except Exception as e:
+            print(f"Failed to add items: {e}")
+            return False
+    
+    def update_items(
+        self,
+        collection_name: str,
+        ids: List[str],
+        documents: Optional[List[str]] = None,
+        metadatas: Optional[List[Dict[str, Any]]] = None,
+        embeddings: Optional[List[List[float]]] = None,
+    ) -> bool:
+        """
+        Update items in a collection.
+        
+        Args:
+            collection_name: Name of collection
+            ids: IDs of items to update
+            documents: New document texts
+            metadatas: New metadata
+            embeddings: New embeddings
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        if not self._client:
+            return False
+        
+        try:
+            # Update existing items
+            # self._client.update(
+            #     collection=collection_name,
+            #     ids=ids,
+            #     documents=documents,
+            #     metadatas=metadatas,
+            #     embeddings=embeddings
+            # )
+            return True
+        except Exception as e:
+            print(f"Failed to update items: {e}")
+            return False
+    
+    def delete_items(
+        self,
+        collection_name: str,
+        ids: Optional[List[str]] = None,
+        where: Optional[Dict[str, Any]] = None,
+    ) -> bool:
+        """
+        Delete items from a collection.
+        
+        Args:
+            collection_name: Name of collection
+            ids: IDs of items to delete
+            where: Metadata filter for items to delete
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        if not self._client:
+            return False
+        
+        try:
+            # Delete items
+            # self._client.delete(
+            #     collection=collection_name,
+            #     ids=ids,
+            #     filter=where
+            # )
+            return True
+        except Exception as e:
+            print(f"Failed to delete items: {e}")
+            return False
+    
+    def delete_collection(self, name: str) -> bool:
+        """
+        Delete an entire collection.
+        
+        Args:
+            name: Collection name
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        if not self._client:
+            return False
+        
+        try:
+            # Delete the collection
+            # self._client.delete_collection(name)
+            return True
+        except Exception as e:
+            print(f"Failed to delete collection: {e}")
+            return False
+    
+    def get_connection_info(self) -> Dict[str, Any]:
+        """
+        Get information about the current connection.
+        
+        Returns:
+            Dictionary with connection details
+        """
+        return {
+            "provider": "Template",  # Replace with your provider name
+            "connected": self.is_connected,
+            # Add provider-specific details here
+        }

vector_inspector/main.py

@@ -0,0 +1,21 @@
+"""Main entry point for Vector Inspector application."""
+
+import sys
+from PySide6.QtWidgets import QApplication
+from vector_inspector.ui.main_window import MainWindow
+
+
+def main():
+    """Launch the Vector Inspector application."""
+    app = QApplication(sys.argv)
+    app.setApplicationName("Vector Inspector")
+    app.setOrganizationName("Vector Inspector")
+    
+    window = MainWindow()
+    window.show()
+    
+    sys.exit(app.exec())
+
+
+if __name__ == "__main__":
+    main()

vector_inspector/services/__init__.py

@@ -0,0 +1 @@
+"""Services for embeddings, visualization, and data processing."""

vector_inspector/services/backup_restore_service.py

@@ -0,0 +1,286 @@
+"""Service for backing up and restoring collections."""
+
+import json
+import zipfile
+from typing import Dict, Any, Optional
+from pathlib import Path
+from datetime import datetime
+import shutil
+
+
+class BackupRestoreService:
+    """Handles backup and restore operations for vector database collections."""
+    
+    @staticmethod
+    def backup_collection(
+        connection,
+        collection_name: str,
+        backup_dir: str,
+        include_embeddings: bool = True
+    ) -> Optional[str]:
+        """
+        Backup a collection to a directory.
+        
+        Args:
+            connection: Vector database connection
+            collection_name: Name of collection to backup
+            backup_dir: Directory to store backups
+            include_embeddings: Whether to include embedding vectors
+            
+        Returns:
+            Path to backup file or None if failed
+        """
+        try:
+            # Create backup directory if it doesn't exist
+            Path(backup_dir).mkdir(parents=True, exist_ok=True)
+            
+            # Get collection info
+            collection_info = connection.get_collection_info(collection_name)
+            if not collection_info:
+                print(f"Failed to get collection info for {collection_name}")
+                return None
+            
+            # Get all items from collection
+            all_data = connection.get_all_items(collection_name)
+            if not all_data or not all_data.get("ids"):
+                print(f"No data to backup from collection {collection_name}")
+                return None
+            
+            # Convert numpy arrays to lists for JSON serialization
+            if "embeddings" in all_data:
+                try:
+                    import numpy as np
+                    if isinstance(all_data["embeddings"], np.ndarray):
+                        all_data["embeddings"] = all_data["embeddings"].tolist()
+                    elif isinstance(all_data["embeddings"], list):
+                        # Convert any numpy arrays in the list
+                        all_data["embeddings"] = [
+                            emb.tolist() if isinstance(emb, np.ndarray) else emb
+                            for emb in all_data["embeddings"]
+                        ]
+                except ImportError:
+                    pass  # numpy not available, assume already lists
+            
+            # Remove embeddings if not needed (to save space)
+            if not include_embeddings and "embeddings" in all_data:
+                del all_data["embeddings"]
+            
+            # Create backup metadata
+            backup_metadata = {
+                "collection_name": collection_name,
+                "backup_timestamp": datetime.now().isoformat(),
+                "item_count": len(all_data["ids"]),
+                "collection_info": collection_info,
+                "include_embeddings": include_embeddings
+            }
+            
+            # Create backup filename with timestamp
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            backup_filename = f"{collection_name}_backup_{timestamp}.zip"
+            backup_path = Path(backup_dir) / backup_filename
+            
+            # Create zip file with data and metadata
+            with zipfile.ZipFile(backup_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+                # Write metadata
+                zipf.writestr('metadata.json', json.dumps(backup_metadata, indent=2))
+                
+                # Write collection data
+                zipf.writestr('data.json', json.dumps(all_data, indent=2))
+            
+            print(f"Backup created: {backup_path}")
+            return str(backup_path)
+            
+        except Exception as e:
+            print(f"Backup failed: {e}")
+            return None
+    
+    @staticmethod
+    def restore_collection(
+        connection,
+        backup_file: str,
+        collection_name: Optional[str] = None,
+        overwrite: bool = False
+    ) -> bool:
+        """
+        Restore a collection from a backup file.
+        
+        Args:
+            connection: Vector database connection
+            backup_file: Path to backup zip file
+            collection_name: Optional new name for restored collection
+            overwrite: Whether to overwrite existing collection
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            # Extract backup
+            with zipfile.ZipFile(backup_file, 'r') as zipf:
+                # Read metadata
+                metadata_str = zipf.read('metadata.json').decode('utf-8')
+                metadata = json.loads(metadata_str)
+                
+                # Read data
+                data_str = zipf.read('data.json').decode('utf-8')
+                data = json.loads(data_str)
+            
+            # Determine collection name
+            restore_collection_name = collection_name or metadata["collection_name"]
+            
+            # Check if collection exists
+            existing_collections = connection.list_collections()
+            if restore_collection_name in existing_collections:
+                if not overwrite:
+                    print(f"Collection {restore_collection_name} already exists. Use overwrite=True to replace it.")
+                    return False
+                else:
+                    # Delete existing collection
+                    connection.delete_collection(restore_collection_name)
+            
+            # Check if this is Qdrant - need to create collection first
+            from vector_inspector.core.connections.qdrant_connection import QdrantConnection
+            if isinstance(connection, QdrantConnection):
+                # Get vector size from collection info or embeddings
+                vector_size = None
+                if metadata.get("collection_info") and "vector_size" in metadata["collection_info"]:
+                    vector_size = metadata["collection_info"]["vector_size"]
+                elif data.get("embeddings") and len(data["embeddings"]) > 0:
+                    vector_size = len(data["embeddings"][0])
+                
+                if not vector_size:
+                    print("Cannot determine vector size for Qdrant collection")
+                    return False
+                
+                # Create collection
+                distance = metadata.get("collection_info", {}).get("distance", "Cosine")
+                if not connection.create_collection(restore_collection_name, vector_size, distance):
+                    print(f"Failed to create collection {restore_collection_name}")
+                    return False
+                
+                # Check if embeddings are missing and need to be generated
+                if not data.get("embeddings"):
+                    print("Embeddings missing in backup. Generating embeddings...")
+                    try:
+                        from sentence_transformers import SentenceTransformer
+                        model = SentenceTransformer("all-MiniLM-L6-v2")
+                        documents = data.get("documents", [])
+                        data["embeddings"] = model.encode(documents, show_progress_bar=True).tolist()
+                    except Exception as e:
+                        print(f"Failed to generate embeddings: {e}")
+                        return False
+                
+                # Convert IDs to Qdrant-compatible format (integers or UUIDs)
+                # Store original IDs in metadata
+                original_ids = data.get("ids", [])
+                qdrant_ids = []
+                metadatas = data.get("metadatas", [])
+                
+                for i, orig_id in enumerate(original_ids):
+                    # Try to convert to integer, otherwise use index
+                    try:
+                        # If it's like "doc_123", extract the number
+                        if isinstance(orig_id, str) and "_" in orig_id:
+                            qdrant_id = int(orig_id.split("_")[-1])
+                        else:
+                            qdrant_id = int(orig_id)
+                    except (ValueError, AttributeError):
+                        # Use index as ID if can't convert
+                        qdrant_id = i
+                    
+                    qdrant_ids.append(qdrant_id)
+                    
+                    # Store original ID in metadata
+                    if i < len(metadatas):
+                        if metadatas[i] is None:
+                            metadatas[i] = {}
+                        metadatas[i]["original_id"] = orig_id
+                    else:
+                        metadatas.append({"original_id": orig_id})
+                
+                data["ids"] = qdrant_ids
+                data["metadatas"] = metadatas
+            
+            # Add items to collection
+            success = connection.add_items(
+                restore_collection_name,
+                documents=data.get("documents", []),
+                metadatas=data.get("metadatas"),
+                ids=data.get("ids"),
+                embeddings=data.get("embeddings")
+            )
+            
+            if success:
+                print(f"Collection '{restore_collection_name}' restored from backup")
+                print(f"Restored {len(data.get('ids', []))} items")
+                return True
+            else:
+                print("Failed to restore collection")
+                return False
+                
+        except Exception as e:
+            print(f"Restore failed: {e}")
+            import traceback
+            traceback.print_exc()
+            return False
+    
+    @staticmethod
+    def list_backups(backup_dir: str) -> list:
+        """
+        List all backup files in a directory.
+        
+        Args:
+            backup_dir: Directory containing backups
+            
+        Returns:
+            List of backup file information dictionaries
+        """
+        try:
+            backup_path = Path(backup_dir)
+            if not backup_path.exists():
+                return []
+            
+            backups = []
+            for backup_file in backup_path.glob("*_backup_*.zip"):
+                try:
+                    # Read metadata from backup
+                    with zipfile.ZipFile(backup_file, 'r') as zipf:
+                        metadata_str = zipf.read('metadata.json').decode('utf-8')
+                        metadata = json.loads(metadata_str)
+                    
+                    backups.append({
+                        "file_path": str(backup_file),
+                        "file_name": backup_file.name,
+                        "collection_name": metadata.get("collection_name", "Unknown"),
+                        "timestamp": metadata.get("backup_timestamp", "Unknown"),
+                        "item_count": metadata.get("item_count", 0),
+                        "file_size": backup_file.stat().st_size
+                    })
+                except Exception:
+                    # Skip invalid backup files
+                    continue
+            
+            # Sort by timestamp (newest first)
+            backups.sort(key=lambda x: x["timestamp"], reverse=True)
+            return backups
+            
+        except Exception as e:
+            print(f"Failed to list backups: {e}")
+            return []
+    
+    @staticmethod
+    def delete_backup(backup_file: str) -> bool:
+        """
+        Delete a backup file.
+        
+        Args:
+            backup_file: Path to backup file to delete
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            Path(backup_file).unlink()
+            return True
+        except Exception as e:
+            print(f"Failed to delete backup: {e}")
+            return False