tribalmemory 0.1.0__py3-none-any.whl

tribalmemory/testing/embedding_utils.py

@@ -0,0 +1,110 @@
+"""Shared embedding utilities for mock implementations.
+
+Provides consistent, deterministic embedding generation for testing.
+"""
+
+import hashlib
+import math
+import random
+import re
+
+
+def hash_to_embedding(text: str, dimensions: int = 1536) -> list[float]:
+    """Convert text to deterministic embedding that preserves semantic similarity.
+    
+    Uses word-level hashing so texts with shared words have similar embeddings.
+    Suitable for basic mock testing scenarios.
+    
+    Args:
+        text: Text to embed.
+        dimensions: Output embedding dimensions.
+        
+    Returns:
+        Normalized embedding vector.
+    """
+    embedding = [0.0] * dimensions
+    
+    def add_term(term: str, weight: float = 1.0):
+        """Add a term's contribution to the embedding."""
+        term_hash = hashlib.sha256(term.encode()).digest()
+        random.seed(int.from_bytes(term_hash[:8], 'big'))
+        for i in range(dimensions):
+            embedding[i] += random.gauss(0, 1) * weight
+    
+    # Normalize text
+    text_lower = text.lower()
+    
+    # Add contribution for each unique word (skip very short words)
+    words = set(re.findall(r'\b\w+\b', text_lower))
+    for word in words:
+        if len(word) > 2:
+            add_term(word, 1.0)
+    
+    # Add short text as a whole (helps exact match queries)
+    if len(text) < 200:
+        add_term(text_lower.strip(), 2.0)
+    
+    # Normalize to unit vector
+    norm = math.sqrt(sum(x * x for x in embedding))
+    if norm == 0:
+        random.seed(42)
+        embedding = [random.gauss(0, 1) for _ in range(dimensions)]
+        norm = math.sqrt(sum(x * x for x in embedding))
+    
+    return [x / norm for x in embedding]
+
+
+def hash_to_embedding_extended(text: str, dimensions: int = 1536) -> list[float]:
+    """Convert text to deterministic embedding with sliding window support.
+    
+    Extended version that uses sliding windows to catch substring matches.
+    Better for tests that need substring similarity detection.
+    
+    Args:
+        text: Text to embed.
+        dimensions: Output embedding dimensions.
+        
+    Returns:
+        Normalized embedding vector.
+    """
+    embedding = [0.0] * dimensions
+    
+    def add_term(term: str, weight: float = 1.0):
+        """Add a term's contribution to the embedding with given weight."""
+        term_hash = hashlib.sha256(term.encode()).digest()
+        random.seed(int.from_bytes(term_hash[:8], 'big'))
+        for i in range(dimensions):
+            embedding[i] += random.gauss(0, 1) * weight
+    
+    # Normalize text
+    text_lower = text.lower()
+    
+    # Add contribution for each unique word
+    words = set(re.findall(r'\b\w+\b', text_lower))
+    for word in words:
+        if len(word) > 2:  # Skip very short words
+            add_term(word, 1.0)
+    
+    # Add sliding windows of characters (catches substrings)
+    # Use windows of size 20, 40, 80 characters
+    for window_size in [20, 40, 80]:
+        seen = set()
+        for i in range(0, len(text_lower) - window_size + 1, window_size // 2):
+            chunk = text_lower[i:i + window_size].strip()
+            if chunk and chunk not in seen:
+                seen.add(chunk)
+                add_term(chunk, 2.0)
+    
+    # For short texts (likely queries), add the whole text as a term
+    # This ensures short exact matches get high similarity
+    if len(text) < 200:
+        add_term(text_lower.strip(), 5.0)
+    
+    # Normalize to unit vector
+    norm = math.sqrt(sum(x * x for x in embedding))
+    if norm == 0:
+        random.seed(42)
+        embedding = [random.gauss(0, 1) for _ in range(dimensions)]
+        norm = math.sqrt(sum(x * x for x in embedding))
+    
+    return [x / norm for x in embedding]

tribalmemory/testing/fixtures.py

@@ -0,0 +1,123 @@
+"""Test data fixtures and loaders."""
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class PreferenceTestCase:
+    """A preference-based test case."""
+    id: str
+    stored_preference: str
+    query: str
+    expected_keywords: list[str]  # Response should contain these
+    context: Optional[str] = None
+    tags: list[str] = field(default_factory=list)
+    negative: bool = False  # If True, this is a negative test case
+
+
+@dataclass
+class ContextTaskTestCase:
+    """A context-dependent task test case."""
+    id: str
+    memories: list[str]  # Memories to store first
+    query: str
+    expected_keywords: list[str]
+    min_memories_referenced: int = 1
+
+
+@dataclass
+class ConsistencyTestCase:
+    """A cross-session consistency test case."""
+    id: str
+    seed_memories: list[str]
+    query_variations: list[str]  # Different ways to ask same thing
+    expected_consistent: bool = True
+
+
+@dataclass
+class TestDataSet:
+    """Complete test data set."""
+    preferences: list[PreferenceTestCase]
+    context_tasks: list[ContextTaskTestCase]
+    consistency: list[ConsistencyTestCase]
+
+
+def load_test_data(data_dir: Path) -> TestDataSet:
+    """Load test data from JSON files."""
+    preferences = []
+    context_tasks = []
+    consistency = []
+    
+    # Load preferences
+    pref_file = data_dir / "preferences.json"
+    if pref_file.exists():
+        with open(pref_file) as f:
+            data = json.load(f)
+            preferences = [PreferenceTestCase(**p) for p in data]
+    
+    # Load context tasks
+    ctx_file = data_dir / "context_tasks.json"
+    if ctx_file.exists():
+        with open(ctx_file) as f:
+            data = json.load(f)
+            context_tasks = [ContextTaskTestCase(**c) for c in data]
+    
+    # Load consistency
+    cons_file = data_dir / "consistency.json"
+    if cons_file.exists():
+        with open(cons_file) as f:
+            data = json.load(f)
+            consistency = [ConsistencyTestCase(**c) for c in data]
+    
+    return TestDataSet(
+        preferences=preferences,
+        context_tasks=context_tasks,
+        consistency=consistency
+    )
+
+
+def save_test_data(dataset: TestDataSet, data_dir: Path):
+    """Save test data to JSON files."""
+    data_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Save preferences
+    with open(data_dir / "preferences.json", "w") as f:
+        json.dump([
+            {
+                "id": p.id,
+                "stored_preference": p.stored_preference,
+                "query": p.query,
+                "expected_keywords": p.expected_keywords,
+                "context": p.context,
+                "tags": p.tags,
+            }
+            for p in dataset.preferences
+        ], f, indent=2)
+    
+    # Save context tasks
+    with open(data_dir / "context_tasks.json", "w") as f:
+        json.dump([
+            {
+                "id": c.id,
+                "memories": c.memories,
+                "query": c.query,
+                "expected_keywords": c.expected_keywords,
+                "min_memories_referenced": c.min_memories_referenced,
+            }
+            for c in dataset.context_tasks
+        ], f, indent=2)
+    
+    # Save consistency
+    with open(data_dir / "consistency.json", "w") as f:
+        json.dump([
+            {
+                "id": c.id,
+                "seed_memories": c.seed_memories,
+                "query_variations": c.query_variations,
+                "expected_consistent": c.expected_consistent,
+            }
+            for c in dataset.consistency
+        ], f, indent=2)

tribalmemory/testing/metrics.py

@@ -0,0 +1,256 @@
+"""Metrics collection and analysis for testing."""
+
+import json
+import math
+import statistics
+import time
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class LatencyMeasurement:
+    """Single latency measurement."""
+    operation: str
+    duration_ms: float
+    timestamp: datetime = field(default_factory=datetime.utcnow)
+    success: bool = True
+    metadata: dict = field(default_factory=dict)
+
+
+class LatencyTracker:
+    """Track and analyze operation latencies."""
+    
+    def __init__(self):
+        self._measurements: list[LatencyMeasurement] = []
+        self._start_times: dict[str, float] = {}
+    
+    def start(self, operation: str) -> str:
+        """Start timing an operation. Returns operation key."""
+        key = f"{operation}_{time.perf_counter_ns()}"
+        self._start_times[key] = time.perf_counter()
+        return key
+    
+    def stop(self, key: str, success: bool = True, **metadata) -> float:
+        """Stop timing and record measurement. Returns duration in ms."""
+        if key not in self._start_times:
+            raise ValueError(f"Unknown operation key: {key}")
+        
+        duration_ms = (time.perf_counter() - self._start_times[key]) * 1000
+        operation = key.rsplit('_', 1)[0]
+        
+        self._measurements.append(LatencyMeasurement(
+            operation=operation,
+            duration_ms=duration_ms,
+            success=success,
+            metadata=metadata
+        ))
+        
+        del self._start_times[key]
+        return duration_ms
+    
+    def record(self, operation: str, duration_ms: float, success: bool = True, **metadata):
+        """Record a measurement directly."""
+        self._measurements.append(LatencyMeasurement(
+            operation=operation,
+            duration_ms=duration_ms,
+            success=success,
+            metadata=metadata
+        ))
+    
+    def get_stats(self, operation: Optional[str] = None) -> dict:
+        """Get statistics for an operation or all operations."""
+        measurements = self._measurements
+        if operation:
+            measurements = [m for m in measurements if m.operation == operation]
+        
+        if not measurements:
+            return {"count": 0}
+        
+        durations = [m.duration_ms for m in measurements]
+        successful = [m for m in measurements if m.success]
+        
+        sorted_durations = sorted(durations)
+        
+        return {
+            "count": len(measurements),
+            "success_rate": len(successful) / len(measurements),
+            "mean_ms": statistics.mean(durations),
+            "median_ms": statistics.median(durations),
+            "stdev_ms": statistics.stdev(durations) if len(durations) > 1 else 0,
+            "min_ms": min(durations),
+            "max_ms": max(durations),
+            "p95_ms": sorted_durations[int(len(sorted_durations) * 0.95)] if len(sorted_durations) > 1 else sorted_durations[0],
+            "p99_ms": sorted_durations[int(len(sorted_durations) * 0.99)] if len(sorted_durations) > 1 else sorted_durations[0],
+        }
+    
+    def clear(self):
+        """Clear all measurements."""
+        self._measurements.clear()
+        self._start_times.clear()
+
+
+class SimilarityCalculator:
+    """Calculate and compare embedding similarities."""
+    
+    @staticmethod
+    def cosine_similarity(a: list[float], b: list[float]) -> float:
+        """Calculate cosine similarity between two vectors."""
+        if len(a) != len(b):
+            raise ValueError("Vectors must have same dimension")
+        
+        dot = sum(x * y for x, y in zip(a, b))
+        norm_a = math.sqrt(sum(x * x for x in a))
+        norm_b = math.sqrt(sum(x * x for x in b))
+        
+        if norm_a == 0 or norm_b == 0:
+            return 0.0
+        
+        return dot / (norm_a * norm_b)
+    
+    @staticmethod
+    def euclidean_distance(a: list[float], b: list[float]) -> float:
+        """Calculate Euclidean distance between two vectors."""
+        if len(a) != len(b):
+            raise ValueError("Vectors must have same dimension")
+        
+        return math.sqrt(sum((x - y) ** 2 for x, y in zip(a, b)))
+    
+    @staticmethod
+    def batch_similarities(
+        query: list[float],
+        candidates: list[list[float]]
+    ) -> list[float]:
+        """Calculate similarities between query and multiple candidates."""
+        return [
+            SimilarityCalculator.cosine_similarity(query, c)
+            for c in candidates
+        ]
+
+
+@dataclass
+class TestResult:
+    """Result of a single test case."""
+    test_id: str
+    test_name: str
+    tier: str
+    passed: bool
+    score: Optional[float] = None
+    threshold: Optional[float] = None
+    duration_ms: float = 0
+    error: Optional[str] = None
+    details: dict = field(default_factory=dict)
+    timestamp: datetime = field(default_factory=datetime.utcnow)
+
+
+class TestResultLogger:
+    """Log and persist test results for analysis."""
+    
+    def __init__(self, output_dir: Optional[Path] = None):
+        self.output_dir = output_dir or Path("test-results")
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self._results: list[TestResult] = []
+        self._run_id = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+    
+    def log(self, result: TestResult):
+        """Log a test result."""
+        self._results.append(result)
+    
+    def get_summary(self) -> dict:
+        """Get summary of all test results."""
+        if not self._results:
+            return {"total": 0}
+        
+        by_tier = {}
+        for r in self._results:
+            if r.tier not in by_tier:
+                by_tier[r.tier] = {"passed": 0, "failed": 0}
+            if r.passed:
+                by_tier[r.tier]["passed"] += 1
+            else:
+                by_tier[r.tier]["failed"] += 1
+        
+        passed = sum(1 for r in self._results if r.passed)
+        failed = len(self._results) - passed
+        
+        return {
+            "run_id": self._run_id,
+            "total": len(self._results),
+            "passed": passed,
+            "failed": failed,
+            "pass_rate": passed / len(self._results),
+            "by_tier": by_tier,
+            "failed_tests": [
+                {"id": r.test_id, "name": r.test_name, "error": r.error}
+                for r in self._results if not r.passed
+            ]
+        }
+    
+    def save(self, filename: Optional[str] = None):
+        """Save results to JSON file."""
+        filename = filename or f"results_{self._run_id}.json"
+        filepath = self.output_dir / filename
+        
+        data = {
+            "run_id": self._run_id,
+            "timestamp": datetime.utcnow().isoformat(),
+            "summary": self.get_summary(),
+            "results": [
+                {
+                    "test_id": r.test_id,
+                    "test_name": r.test_name,
+                    "tier": r.tier,
+                    "passed": r.passed,
+                    "score": r.score,
+                    "threshold": r.threshold,
+                    "duration_ms": r.duration_ms,
+                    "error": r.error,
+                    "details": r.details,
+                    "timestamp": r.timestamp.isoformat(),
+                }
+                for r in self._results
+            ]
+        }
+        
+        with open(filepath, "w") as f:
+            json.dump(data, f, indent=2)
+        
+        return filepath
+    
+    def compare_to_baseline(self, baseline_path: Path) -> dict:
+        """Compare current results to a baseline."""
+        with open(baseline_path) as f:
+            baseline = json.load(f)
+        
+        current_by_id = {r.test_id: r for r in self._results}
+        baseline_by_id = {r["test_id"]: r for r in baseline["results"]}
+        
+        regressions = []
+        improvements = []
+        new_tests = []
+        
+        for test_id, result in current_by_id.items():
+            if test_id not in baseline_by_id:
+                new_tests.append(test_id)
+                continue
+            
+            baseline_result = baseline_by_id[test_id]
+            
+            if result.passed and not baseline_result["passed"]:
+                improvements.append(test_id)
+            elif not result.passed and baseline_result["passed"]:
+                regressions.append(test_id)
+        
+        return {
+            "regressions": regressions,
+            "improvements": improvements,
+            "new_tests": new_tests,
+            "baseline_pass_rate": baseline["summary"]["pass_rate"],
+            "current_pass_rate": self.get_summary()["pass_rate"],
+        }
+    
+    def clear(self):
+        """Clear all results."""
+        self._results.clear()