vector-inspector 0.2.0__py3-none-any.whl

vector_inspector/services/filter_service.py

@@ -0,0 +1,72 @@
+"""Service for applying client-side filters to data."""
+
+from typing import Dict, Any, List
+
+
+def apply_client_side_filters(data: Dict[str, Any], filters: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """
+    Apply client-side filters to fetched data.
+    
+    Args:
+        data: Data dictionary with ids, documents, metadatas, etc.
+        filters: List of client-side filter dictionaries
+        
+    Returns:
+        Filtered data dictionary
+    """
+    if not filters:
+        return data
+    
+    ids = data.get("ids", [])
+    documents = data.get("documents", [])
+    metadatas = data.get("metadatas", [])
+    embeddings = data.get("embeddings", [])
+    
+    # Track which indices to keep
+    keep_indices = []
+    
+    for i in range(len(ids)):
+        # Check if this item passes all client-side filters
+        passes = True
+        
+        for filt in filters:
+            field = filt.get("field", "")
+            op = filt.get("op", "")
+            value = filt.get("value", "")
+            
+            # Special handling for document field
+            if field.lower() == "document":
+                item_value = documents[i] if i < len(documents) else ""
+            else:
+                # Get from metadata
+                metadata = metadatas[i] if i < len(metadatas) else {}
+                item_value = metadata.get(field, "")
+            
+            # Convert to string for text operations
+            item_value_str = str(item_value).lower()
+            search_value = str(value).lower()
+            
+            # Apply operator
+            if op == "contains":
+                if search_value not in item_value_str:
+                    passes = False
+                    break
+            elif op == "not_contains":
+                if search_value in item_value_str:
+                    passes = False
+                    break
+        
+        if passes:
+            keep_indices.append(i)
+    
+    # Filter the data
+    filtered_data = {
+        "ids": [ids[i] for i in keep_indices],
+        "documents": [documents[i] for i in keep_indices if i < len(documents)],
+        "metadatas": [metadatas[i] for i in keep_indices if i < len(metadatas)],
+    }
+    
+    if embeddings:
+        filtered_data["embeddings"] = [embeddings[i] for i in keep_indices if i < len(embeddings)]
+    
+    return filtered_data

vector_inspector/services/import_export_service.py

@@ -0,0 +1,287 @@
+"""Service for importing and exporting collection data."""
+
+import json
+import csv
+from typing import Dict, Any, List, Optional
+from pathlib import Path
+import pandas as pd
+
+
+class ImportExportService:
+    """Handles import/export operations for vector database collections."""
+    
+    @staticmethod
+    def export_to_json(data: Dict[str, Any], file_path: str) -> bool:
+        """
+        Export collection data to JSON format.
+        
+        Args:
+            data: Collection data dictionary with ids, documents, metadatas, embeddings
+            file_path: Path to save JSON file
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            # Structure data for export
+            export_data = []
+            ids = data.get("ids", [])
+            documents = data.get("documents", [])
+            metadatas = data.get("metadatas", [])
+            embeddings = data.get("embeddings", [])
+            
+            for i, item_id in enumerate(ids):
+                item = {
+                    "id": item_id,
+                    "document": documents[i] if i < len(documents) else None,
+                    "metadata": metadatas[i] if i < len(metadatas) else {},
+                }
+                # Optionally include embeddings
+                if embeddings and i < len(embeddings):
+                    item["embedding"] = embeddings[i]
+                export_data.append(item)
+            
+            # Write to file
+            with open(file_path, 'w', encoding='utf-8') as f:
+                json.dump(export_data, f, indent=2, ensure_ascii=False)
+                
+            return True
+        except Exception as e:
+            print(f"Export to JSON failed: {e}")
+            return False
+    
+    @staticmethod
+    def export_to_csv(data: Dict[str, Any], file_path: str, include_embeddings: bool = False) -> bool:
+        """
+        Export collection data to CSV format.
+        
+        Args:
+            data: Collection data dictionary
+            file_path: Path to save CSV file
+            include_embeddings: Whether to include embedding vectors
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            ids = data.get("ids", [])
+            documents = data.get("documents", [])
+            metadatas = data.get("metadatas", [])
+            embeddings = data.get("embeddings", [])
+            
+            # Prepare rows
+            rows = []
+            for i, item_id in enumerate(ids):
+                row = {
+                    "id": item_id,
+                    "document": documents[i] if i < len(documents) else "",
+                }
+                
+                # Add metadata fields
+                if i < len(metadatas) and metadatas[i]:
+                    for key, value in metadatas[i].items():
+                        row[f"metadata_{key}"] = value
+                
+                # Optionally add embeddings
+                if include_embeddings and embeddings and i < len(embeddings):
+                    row["embedding"] = json.dumps(embeddings[i])
+                    
+                rows.append(row)
+            
+            # Convert to DataFrame and save
+            df = pd.DataFrame(rows)
+            df.to_csv(file_path, index=False, encoding='utf-8')
+            
+            return True
+        except Exception as e:
+            print(f"Export to CSV failed: {e}")
+            return False
+    
+    @staticmethod
+    def export_to_parquet(data: Dict[str, Any], file_path: str) -> bool:
+        """
+        Export collection data to Parquet format.
+        
+        Args:
+            data: Collection data dictionary
+            file_path: Path to save Parquet file
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        try:
+            ids = data.get("ids", [])
+            documents = data.get("documents", [])
+            metadatas = data.get("metadatas", [])
+            embeddings = data.get("embeddings", [])
+            
+            # Prepare data for DataFrame
+            df_data = {
+                "id": ids,
+                "document": documents if documents else [None] * len(ids),
+            }
+            
+            # Add metadata fields as separate columns
+            if metadatas and metadatas[0]:
+                for key in metadatas[0].keys():
+                    df_data[f"metadata_{key}"] = [m.get(key) if m else None for m in metadatas]
+            
+            # Add embeddings as a column
+            if embeddings:
+                df_data["embedding"] = embeddings
+            
+            # Create DataFrame and save
+            df = pd.DataFrame(df_data)
+            df.to_parquet(file_path, index=False, engine='pyarrow')
+            
+            return True
+        except Exception as e:
+            print(f"Export to Parquet failed: {e}")
+            return False
+    
+    @staticmethod
+    def import_from_json(file_path: str) -> Optional[Dict[str, Any]]:
+        """
+        Import collection data from JSON format.
+        
+        Args:
+            file_path: Path to JSON file
+            
+        Returns:
+            Dictionary with ids, documents, metadatas, embeddings or None if failed
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            # Parse data
+            ids = []
+            documents = []
+            metadatas = []
+            embeddings = []
+            
+            for item in data:
+                ids.append(item.get("id", ""))
+                documents.append(item.get("document", ""))
+                metadatas.append(item.get("metadata", {}))
+                if "embedding" in item:
+                    embeddings.append(item["embedding"])
+            
+            result = {
+                "ids": ids,
+                "documents": documents,
+                "metadatas": metadatas,
+            }
+            
+            if embeddings:
+                result["embeddings"] = embeddings
+                
+            return result
+            
+        except Exception as e:
+            print(f"Import from JSON failed: {e}")
+            return None
+    
+    @staticmethod
+    def import_from_csv(file_path: str) -> Optional[Dict[str, Any]]:
+        """
+        Import collection data from CSV format.
+        
+        Args:
+            file_path: Path to CSV file
+            
+        Returns:
+            Dictionary with ids, documents, metadatas, embeddings or None if failed
+        """
+        try:
+            df = pd.read_csv(file_path, encoding='utf-8')
+            
+            ids = df["id"].tolist()
+            documents = df["document"].tolist() if "document" in df.columns else [""] * len(ids)
+            
+            # Extract metadata columns
+            metadata_cols = [col for col in df.columns if col.startswith("metadata_")]
+            metadatas = []
+            
+            for idx in range(len(ids)):
+                metadata = {}
+                for col in metadata_cols:
+                    key = col.replace("metadata_", "")
+                    value = df.loc[idx, col]
+                    if pd.notna(value):
+                        metadata[key] = value
+                metadatas.append(metadata)
+            
+            # Extract embeddings if present
+            embeddings = []
+            if "embedding" in df.columns:
+                for emb_str in df["embedding"]:
+                    if pd.notna(emb_str):
+                        embeddings.append(json.loads(emb_str))
+                    else:
+                        embeddings.append([])
+            
+            result = {
+                "ids": ids,
+                "documents": documents,
+                "metadatas": metadatas,
+            }
+            
+            if embeddings:
+                result["embeddings"] = embeddings
+                
+            return result
+            
+        except Exception as e:
+            print(f"Import from CSV failed: {e}")
+            return None
+    
+    @staticmethod
+    def import_from_parquet(file_path: str) -> Optional[Dict[str, Any]]:
+        """
+        Import collection data from Parquet format.
+        
+        Args:
+            file_path: Path to Parquet file
+            
+        Returns:
+            Dictionary with ids, documents, metadatas, embeddings or None if failed
+        """
+        try:
+            df = pd.read_parquet(file_path, engine='pyarrow')
+            
+            ids = df["id"].tolist()
+            documents = df["document"].tolist() if "document" in df.columns else [""] * len(ids)
+            
+            # Extract metadata columns
+            metadata_cols = [col for col in df.columns if col.startswith("metadata_")]
+            metadatas = []
+            
+            for idx in range(len(ids)):
+                metadata = {}
+                for col in metadata_cols:
+                    key = col.replace("metadata_", "")
+                    value = df.loc[idx, col]
+                    if pd.notna(value):
+                        metadata[key] = value
+                metadatas.append(metadata)
+            
+            # Extract embeddings if present
+            embeddings = []
+            if "embedding" in df.columns:
+                embeddings = df["embedding"].tolist()
+            
+            result = {
+                "ids": ids,
+                "documents": documents,
+                "metadatas": metadatas,
+            }
+            
+            if embeddings:
+                result["embeddings"] = embeddings
+                
+            return result
+            
+        except Exception as e:
+            print(f"Import from Parquet failed: {e}")
+            return None

vector_inspector/services/settings_service.py

@@ -0,0 +1,60 @@
+"""Service for persisting application settings."""
+
+import json
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+
+class SettingsService:
+    """Handles loading and saving application settings."""
+    
+    def __init__(self):
+        """Initialize settings service."""
+        self.settings_dir = Path.home() / ".vector-viewer"
+        self.settings_file = self.settings_dir / "settings.json"
+        self.settings: Dict[str, Any] = {}
+        self._load_settings()
+    
+    def _load_settings(self):
+        """Load settings from file."""
+        try:
+            if self.settings_file.exists():
+                with open(self.settings_file, 'r', encoding='utf-8') as f:
+                    self.settings = json.load(f)
+        except Exception as e:
+            print(f"Failed to load settings: {e}")
+            self.settings = {}
+    
+    def _save_settings(self):
+        """Save settings to file."""
+        try:
+            # Create settings directory if it doesn't exist
+            self.settings_dir.mkdir(parents=True, exist_ok=True)
+            
+            with open(self.settings_file, 'w', encoding='utf-8') as f:
+                json.dump(self.settings, f, indent=2, ensure_ascii=False)
+        except Exception as e:
+            print(f"Failed to save settings: {e}")
+    
+    def get_last_connection(self) -> Optional[Dict[str, Any]]:
+        """Get the last connection configuration."""
+        return self.settings.get("last_connection")
+    
+    def save_last_connection(self, config: Dict[str, Any]):
+        """Save the last connection configuration."""
+        self.settings["last_connection"] = config
+        self._save_settings()
+    
+    def get(self, key: str, default: Any = None) -> Any:
+        """Get a setting value."""
+        return self.settings.get(key, default)
+    
+    def set(self, key: str, value: Any):
+        """Set a setting value."""
+        self.settings[key] = value
+        self._save_settings()
+    
+    def clear(self):
+        """Clear all settings."""
+        self.settings = {}
+        self._save_settings()

vector_inspector/services/visualization_service.py

@@ -0,0 +1,116 @@
+"""Visualization service for dimensionality reduction."""
+
+from typing import Optional, List, Tuple
+import numpy as np
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+import umap
+
+
+class VisualizationService:
+    """Service for vector dimensionality reduction and visualization."""
+    
+    @staticmethod
+    def reduce_dimensions(
+        embeddings: List[List[float]],
+        method: str = "pca",
+        n_components: int = 2,
+        **kwargs
+    ) -> Optional[np.ndarray]:
+        """
+        Reduce dimensionality of embeddings.
+        
+        Args:
+            embeddings: List of embedding vectors
+            method: Reduction method ('pca', 'tsne', or 'umap')
+            n_components: Target number of dimensions (2 or 3)
+            **kwargs: Additional method-specific parameters
+            
+        Returns:
+            Reduced embeddings as numpy array, or None if failed
+        """
+        if embeddings is None or len(embeddings) == 0:
+            return None
+            
+        try:
+            X = np.array(embeddings)
+            
+            if method == "pca":
+                reducer = PCA(n_components=n_components)
+                reduced = reducer.fit_transform(X)
+                
+            elif method == "tsne":
+                perplexity = kwargs.get("perplexity", min(30, len(embeddings) - 1))
+                reducer = TSNE(
+                    n_components=n_components,
+                    perplexity=perplexity,
+                    random_state=42
+                )
+                reduced = reducer.fit_transform(X)
+                
+            elif method == "umap":
+                n_neighbors = kwargs.get("n_neighbors", min(15, len(embeddings) - 1))
+                reducer = umap.UMAP(
+                    n_components=n_components,
+                    n_neighbors=n_neighbors,
+                    random_state=42
+                )
+                reduced = reducer.fit_transform(X)
+                
+            else:
+                print(f"Unknown method: {method}")
+                return None
+                
+            return reduced
+            
+        except Exception as e:
+            print(f"Dimensionality reduction failed: {e}")
+            return None
+    
+    @staticmethod
+    def prepare_plot_data(
+        reduced_embeddings: np.ndarray,
+        labels: Optional[List[str]] = None,
+        metadata: Optional[List[dict]] = None,
+        color_by: Optional[str] = None
+    ) -> Tuple[np.ndarray, List[str], List[str]]:
+        """
+        Prepare data for plotting.
+        
+        Args:
+            reduced_embeddings: Reduced dimension embeddings
+            labels: Text labels for each point
+            metadata: Metadata dictionaries for each point
+            color_by: Metadata field to use for coloring
+            
+        Returns:
+            Tuple of (embeddings, labels, colors)
+        """
+        n_points = len(reduced_embeddings)
+        
+        # Prepare labels
+        if labels is None:
+            labels = [f"Point {i}" for i in range(n_points)]
+        
+        # Prepare colors
+        colors = ["blue"] * n_points
+        if color_by and metadata:
+            unique_values = set()
+            values = []
+            for meta in metadata:
+                value = meta.get(color_by, "unknown")
+                values.append(str(value))
+                unique_values.add(str(value))
+            
+            # Map values to colors
+            color_map = {}
+            color_palette = [
+                "red", "blue", "green", "orange", "purple",
+                "cyan", "magenta", "yellow", "pink", "brown"
+            ]
+            for i, val in enumerate(sorted(unique_values)):
+                color_map[val] = color_palette[i % len(color_palette)]
+            
+            colors = [color_map[val] for val in values]
+        
+        return reduced_embeddings, labels, colors

vector_inspector/ui/__init__.py

@@ -0,0 +1 @@
+"""User interface components and views."""

vector_inspector/ui/components/__init__.py

@@ -0,0 +1 @@
+"""Reusable UI components."""