vector-inspector 0.3.2__py3-none-any.whl → 0.3.4__py3-none-any.whl

vector_inspector/services/backup_restore_service.py

@@ -1,12 +1,14 @@
 """Service for backing up and restoring collections."""
 
 import json
-import zipfile
 from typing import Dict, Any, Optional
 from pathlib import Path
 from datetime import datetime
 import shutil
 
+from ..core.logging import log_info, log_error, log_debug
+from .backup_helpers import write_backup_zip, read_backup_zip, normalize_embeddings
+
 
 class BackupRestoreService:
     """Handles backup and restore operations for vector database collections."""
@@ -31,67 +33,41 @@ class BackupRestoreService:
             Path to backup file or None if failed
         """
         try:
-            # Create backup directory if it doesn't exist
             Path(backup_dir).mkdir(parents=True, exist_ok=True)
-            
-            # Get collection info
+
             collection_info = connection.get_collection_info(collection_name)
             if not collection_info:
-                print(f"Failed to get collection info for {collection_name}")
+                log_error("Failed to get collection info for %s", collection_name)
                 return None
-            
-            # Get all items from collection
+
             all_data = connection.get_all_items(collection_name)
             if not all_data or not all_data.get("ids"):
-                print(f"No data to backup from collection {collection_name}")
+                log_info("No data to backup from collection %s", collection_name)
                 return None
-            
-            # Convert numpy arrays to lists for JSON serialization
-            if "embeddings" in all_data:
-                try:
-                    import numpy as np
-                    if isinstance(all_data["embeddings"], np.ndarray):
-                        all_data["embeddings"] = all_data["embeddings"].tolist()
-                    elif isinstance(all_data["embeddings"], list):
-                        # Convert any numpy arrays in the list
-                        all_data["embeddings"] = [
-                            emb.tolist() if isinstance(emb, np.ndarray) else emb
-                            for emb in all_data["embeddings"]
-                        ]
-                except ImportError:
-                    pass  # numpy not available, assume already lists
-            
-            # Remove embeddings if not needed (to save space)
+
+            # Normalize embeddings to plain lists
+            all_data = normalize_embeddings(all_data)
+
             if not include_embeddings and "embeddings" in all_data:
                 del all_data["embeddings"]
-            
-            # Create backup metadata
+
             backup_metadata = {
                 "collection_name": collection_name,
                 "backup_timestamp": datetime.now().isoformat(),
                 "item_count": len(all_data["ids"]),
                 "collection_info": collection_info,
-                "include_embeddings": include_embeddings
+                "include_embeddings": include_embeddings,
             }
-            
-            # Create backup filename with timestamp
+
             timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
             backup_filename = f"{collection_name}_backup_{timestamp}.zip"
             backup_path = Path(backup_dir) / backup_filename
-            
-            # Create zip file with data and metadata
-            with zipfile.ZipFile(backup_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-                # Write metadata
-                zipf.writestr('metadata.json', json.dumps(backup_metadata, indent=2))
-                
-                # Write collection data
-                zipf.writestr('data.json', json.dumps(all_data, indent=2))
-            
-            print(f"Backup created: {backup_path}")
+
+            write_backup_zip(backup_path, backup_metadata, all_data)
+            log_info("Backup created: %s", backup_path)
             return str(backup_path)
-            
         except Exception as e:
-            print(f"Backup failed: {e}")
+            log_error("Backup failed: %s", e)
             return None
     
     @staticmethod
@@ -113,104 +89,60 @@ class BackupRestoreService:
         Returns:
             True if successful, False otherwise
         """
+        restore_collection_name = None
         try:
-            # Extract backup
-            with zipfile.ZipFile(backup_file, 'r') as zipf:
-                # Read metadata
-                metadata_str = zipf.read('metadata.json').decode('utf-8')
-                metadata = json.loads(metadata_str)
-                
-                # Read data
-                data_str = zipf.read('data.json').decode('utf-8')
-                data = json.loads(data_str)
-            
-            # Determine collection name
-            restore_collection_name = collection_name or metadata["collection_name"]
-            
-            # Check if collection exists
+            metadata, data = read_backup_zip(backup_file)
+            restore_collection_name = collection_name or metadata.get("collection_name")
+
             existing_collections = connection.list_collections()
             if restore_collection_name in existing_collections:
                 if not overwrite:
-                    print(f"Collection {restore_collection_name} already exists. Use overwrite=True to replace it.")
+                    log_info("Collection %s already exists. Use overwrite=True to replace it.", restore_collection_name)
                     return False
                 else:
-                    # Delete existing collection
                     connection.delete_collection(restore_collection_name)
-            
-            # Check if this is Qdrant - need to create collection first
-            from vector_inspector.core.connections.qdrant_connection import QdrantConnection
-            if isinstance(connection, QdrantConnection):
-                # Get vector size from collection info or embeddings
-                vector_size = None
-                if metadata.get("collection_info") and "vector_size" in metadata["collection_info"]:
-                    vector_size = metadata["collection_info"]["vector_size"]
-                elif data.get("embeddings") and len(data["embeddings"]) > 0:
-                    vector_size = len(data["embeddings"][0])
-                
-                if not vector_size:
-                    print("Cannot determine vector size for Qdrant collection")
-                    return False
-                
-                # Create collection
-                distance = metadata.get("collection_info", {}).get("distance", "Cosine")
-                if not connection.create_collection(restore_collection_name, vector_size, distance):
-                    print(f"Failed to create collection {restore_collection_name}")
+
+            # Provider-specific preparation hook
+            if hasattr(connection, "prepare_restore"):
+                ok = connection.prepare_restore(metadata, data)
+                if not ok:
+                    log_error("Provider prepare_restore failed for %s", restore_collection_name)
                     return False
-                
-                # Check if embeddings are missing and need to be generated
-                if not data.get("embeddings"):
-                    print("Embeddings missing in backup. Generating embeddings...")
-                    try:
-                        from sentence_transformers import SentenceTransformer
-                        model = SentenceTransformer("all-MiniLM-L6-v2")
-                        documents = data.get("documents", [])
-                        data["embeddings"] = model.encode(documents, show_progress_bar=True).tolist()
-                    except Exception as e:
-                        print(f"Failed to generate embeddings: {e}")
-                        return False
-                
-                # Keep IDs as strings - Qdrant's _to_uuid method handles conversion
-                # Just ensure all IDs are strings
-                original_ids = data.get("ids", [])
-                data["ids"] = [str(id_val) for id_val in original_ids]
-            
-            # Add items to collection
+
+            # Ensure embeddings normalized
+            data = normalize_embeddings(data)
+
             success = connection.add_items(
                 restore_collection_name,
                 documents=data.get("documents", []),
                 metadatas=data.get("metadatas"),
                 ids=data.get("ids"),
-                embeddings=data.get("embeddings")
+                embeddings=data.get("embeddings"),
             )
-            
+
             if success:
-                print(f"Collection '{restore_collection_name}' restored from backup")
-                print(f"Restored {len(data.get('ids', []))} items")
+                log_info("Collection '%s' restored from backup", restore_collection_name)
+                log_info("Restored %d items", len(data.get("ids", [])))
                 return True
-            else:
-                print("Failed to restore collection")
-                # Clean up partially created collection
-                try:
-                    if restore_collection_name in connection.list_collections():
-                        print(f"Cleaning up failed restore: deleting collection '{restore_collection_name}'")
-                        connection.delete_collection(restore_collection_name)
-                except Exception as cleanup_error:
-                    print(f"Warning: Failed to clean up collection: {cleanup_error}")
-                return False
-                
-        except Exception as e:
-            print(f"Restore failed: {e}")
-            import traceback
-            traceback.print_exc()
-            
-            # Clean up partially created collection
+
+            # Failure: attempt cleanup
+            log_error("Failed to restore collection %s", restore_collection_name)
             try:
                 if restore_collection_name in connection.list_collections():
-                    print(f"Cleaning up failed restore: deleting collection '{restore_collection_name}'")
+                    log_info("Cleaning up failed restore: deleting collection '%s'", restore_collection_name)
                     connection.delete_collection(restore_collection_name)
             except Exception as cleanup_error:
-                print(f"Warning: Failed to clean up collection: {cleanup_error}")
-            
+                log_error("Warning: Failed to clean up collection: %s", cleanup_error)
+            return False
+
+        except Exception as e:
+            log_error("Restore failed: %s", e)
+            try:
+                if restore_collection_name and restore_collection_name in connection.list_collections():
+                    log_info("Cleaning up failed restore: deleting collection '%s'", restore_collection_name)
+                    connection.delete_collection(restore_collection_name)
+            except Exception as cleanup_error:
+                log_error("Warning: Failed to clean up collection: %s", cleanup_error)
             return False
     
     @staticmethod
@@ -224,38 +156,27 @@ class BackupRestoreService:
         Returns:
             List of backup file information dictionaries
         """
-        try:
-            backup_path = Path(backup_dir)
-            if not backup_path.exists():
-                return []
-            
-            backups = []
-            for backup_file in backup_path.glob("*_backup_*.zip"):
-                try:
-                    # Read metadata from backup
-                    with zipfile.ZipFile(backup_file, 'r') as zipf:
-                        metadata_str = zipf.read('metadata.json').decode('utf-8')
-                        metadata = json.loads(metadata_str)
-                    
-                    backups.append({
-                        "file_path": str(backup_file),
-                        "file_name": backup_file.name,
-                        "collection_name": metadata.get("collection_name", "Unknown"),
-                        "timestamp": metadata.get("backup_timestamp", "Unknown"),
-                        "item_count": metadata.get("item_count", 0),
-                        "file_size": backup_file.stat().st_size
-                    })
-                except Exception:
-                    # Skip invalid backup files
-                    continue
-            
-            # Sort by timestamp (newest first)
-            backups.sort(key=lambda x: x["timestamp"], reverse=True)
-            return backups
-            
-        except Exception as e:
-            print(f"Failed to list backups: {e}")
+        backup_path = Path(backup_dir)
+        if not backup_path.exists():
             return []
+
+        backups = []
+        for backup_file in backup_path.glob("*_backup_*.zip"):
+            try:
+                metadata, _ = read_backup_zip(backup_file)
+                backups.append({
+                    "file_path": str(backup_file),
+                    "file_name": backup_file.name,
+                    "collection_name": metadata.get("collection_name", "Unknown"),
+                    "timestamp": metadata.get("backup_timestamp", "Unknown"),
+                    "item_count": metadata.get("item_count", 0),
+                    "file_size": backup_file.stat().st_size,
+                })
+            except Exception:
+                continue
+
+        backups.sort(key=lambda x: x["timestamp"], reverse=True)
+        return backups
     
     @staticmethod
     def delete_backup(backup_file: str) -> bool:
@@ -272,5 +193,5 @@ class BackupRestoreService:
             Path(backup_file).unlink()
             return True
         except Exception as e:
-            print(f"Failed to delete backup: {e}")
+            log_error("Failed to delete backup: %s", e)
             return False

vector_inspector/services/credential_service.py

@@ -2,123 +2,117 @@
 
 from typing import Optional
 import json
+from vector_inspector.core.logging import log_info, log_error
 
 
 class CredentialService:
     """Handles secure storage and retrieval of credentials using system keychains.
-    
+
     Falls back to in-memory storage if keyring is not available (not recommended for production).
     """
-    
+
     SERVICE_NAME = "vector-inspector"
-    
+
     def __init__(self):
         """Initialize credential service with keyring if available."""
         self._use_keyring = False
         self._memory_store = {}  # Fallback in-memory storage
-        
+
         try:
             import keyring
+
             self._keyring = keyring
             self._use_keyring = True
         except ImportError:
-            print("Warning: keyring module not available. Credentials will not be persisted securely.")
+            log_info(
+                "Warning: keyring module not available. Credentials will not be persisted securely."
+            )
             self._keyring = None
-    
+
     def store_credentials(self, profile_id: str, credentials: dict) -> bool:
         """
         Store credentials for a profile.
-        
+
         Args:
             profile_id: Unique profile identifier
             credentials: Dictionary of credential data (api_key, password, etc.)
-            
+
         Returns:
             True if successful, False otherwise
         """
         try:
             credential_key = f"profile:{profile_id}"
             credential_json = json.dumps(credentials)
-            
+
             if self._use_keyring:
-                self._keyring.set_password(
-                    self.SERVICE_NAME,
-                    credential_key,
-                    credential_json
-                )
+                self._keyring.set_password(self.SERVICE_NAME, credential_key, credential_json)
             else:
                 # Fallback to in-memory (not persistent)
                 self._memory_store[credential_key] = credential_json
-            
+
             return True
         except Exception as e:
-            print(f"Failed to store credentials: {e}")
+            log_error("Failed to store credentials: %s", e)
             return False
-    
+
     def get_credentials(self, profile_id: str) -> Optional[dict]:
         """
         Retrieve credentials for a profile.
-        
+
         Args:
             profile_id: Unique profile identifier
-            
+
         Returns:
             Dictionary of credential data, or None if not found
         """
         try:
             credential_key = f"profile:{profile_id}"
-            
+
             if self._use_keyring:
-                credential_json = self._keyring.get_password(
-                    self.SERVICE_NAME,
-                    credential_key
-                )
+                credential_json = self._keyring.get_password(self.SERVICE_NAME, credential_key)
             else:
                 # Fallback to in-memory
                 credential_json = self._memory_store.get(credential_key)
-            
+
             if credential_json:
                 return json.loads(credential_json)
             return None
         except Exception as e:
-            print(f"Failed to retrieve credentials: {e}")
+            log_error("Failed to retrieve credentials: %s", e)
             return None
-    
+
     def delete_credentials(self, profile_id: str) -> bool:
         """
         Delete stored credentials for a profile.
-        
+
         Args:
             profile_id: Unique profile identifier
-            
+
         Returns:
             True if successful, False otherwise
         """
         try:
             credential_key = f"profile:{profile_id}"
-            
+
             if self._use_keyring:
                 try:
-                    self._keyring.delete_password(
-                        self.SERVICE_NAME,
-                        credential_key
-                    )
+                    self._keyring.delete_password(self.SERVICE_NAME, credential_key)
                 except self._keyring.errors.PasswordDeleteError:
                     # Credential doesn't exist, that's okay
                     pass
             else:
                 # Fallback to in-memory
                 self._memory_store.pop(credential_key, None)
-            
+
             return True
         except Exception as e:
-            print(f"Failed to delete credentials: {e}")
+            log_error("Failed to delete credentials: %s", e)
             return False
-    
+
     def is_keyring_available(self) -> bool:
         """Check if system keyring is available."""
         return self._use_keyring
-    
+
     def clear_all_credentials(self):
         """Clear all stored credentials. Use with caution!"""
         if not self._use_keyring:
@@ -126,5 +120,4 @@ class CredentialService:
         else:
             # For keyring, we'd need to track all profile IDs
             # This is typically not needed, but can be implemented if required
-            print("Warning: clear_all_credentials not implemented for keyring backend")
-
+            log_info("Warning: clear_all_credentials not implemented for keyring backend")