hindsight-api 0.0.21__tar.gz → 0.1.0__tar.gz

{hindsight_api-0.0.21 → hindsight_api-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hindsight-api
-Version: 0.0.21
+Version: 0.1.0
 Summary: Temporal + Semantic + Entity Memory System for AI agents using PostgreSQL
 Requires-Python: >=3.11
 Requires-Dist: alembic>=1.17.1
@@ -23,7 +23,7 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: python-dateutil>=2.8.0
 Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: rich>=13.0.0
-Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: sentence-transformers>=3.0.0
 Requires-Dist: sqlalchemy>=2.0.44
 Requires-Dist: tiktoken>=0.12.0
 Requires-Dist: torch>=2.0.0
@@ -36,7 +36,6 @@ Requires-Dist: pytest-asyncio>=0.21.0; extra == 'test'
 Requires-Dist: pytest-timeout>=2.4.0; extra == 'test'
 Requires-Dist: pytest-xdist>=3.0.0; extra == 'test'
 Requires-Dist: pytest>=7.0.0; extra == 'test'
-Requires-Dist: testcontainers[postgres]>=4.0.0; extra == 'test'
 Description-Content-Type: text/markdown
 
 # Memory 

hindsight_api-0.1.0/alembic/versions/e0a1b2c3d4e5_disposition_to_3_traits.py

@@ -0,0 +1,62 @@
+"""disposition_to_3_traits
+
+Revision ID: e0a1b2c3d4e5
+Revises: rename_personality
+Create Date: 2024-12-08
+
+Migrate disposition traits from Big Five (openness, conscientiousness, extraversion,
+agreeableness, neuroticism, bias_strength with 0-1 float values) to the new 3-trait
+system (skepticism, literalism, empathy with 1-5 integer values).
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'e0a1b2c3d4e5'
+down_revision: Union[str, Sequence[str], None] = 'rename_personality'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Convert Big Five disposition to 3-trait disposition."""
+    conn = op.get_bind()
+
+    # Update all existing banks to use the new disposition format
+    # Convert from old format to new format with reasonable mappings:
+    # - skepticism: derived from inverse of agreeableness (skeptical people are less agreeable)
+    # - literalism: derived from conscientiousness (detail-oriented people are more literal)
+    # - empathy: derived from agreeableness + inverse of neuroticism
+    # Default all to 3 (neutral) for simplicity
+    conn.execute(sa.text("""
+        UPDATE banks
+        SET disposition = '{"skepticism": 3, "literalism": 3, "empathy": 3}'::jsonb
+        WHERE disposition IS NOT NULL
+    """))
+
+    # Update the default for new banks
+    conn.execute(sa.text("""
+        ALTER TABLE banks
+        ALTER COLUMN disposition SET DEFAULT '{"skepticism": 3, "literalism": 3, "empathy": 3}'::jsonb
+    """))
+
+
+def downgrade() -> None:
+    """Convert back to Big Five disposition."""
+    conn = op.get_bind()
+
+    # Revert to Big Five format with default values
+    conn.execute(sa.text("""
+        UPDATE banks
+        SET disposition = '{"openness": 0.5, "conscientiousness": 0.5, "extraversion": 0.5, "agreeableness": 0.5, "neuroticism": 0.5, "bias_strength": 0.5}'::jsonb
+        WHERE disposition IS NOT NULL
+    """))
+
+    # Update the default for new banks
+    conn.execute(sa.text("""
+        ALTER TABLE banks
+        ALTER COLUMN disposition SET DEFAULT '{"openness": 0.5, "conscientiousness": 0.5, "extraversion": 0.5, "agreeableness": 0.5, "neuroticism": 0.5, "bias_strength": 0.5}'::jsonb
+    """))

{hindsight_api-0.0.21 → hindsight_api-0.1.0}/hindsight_api/api/__init__.py

@@ -17,18 +17,17 @@ def create_app(
     http_api_enabled: bool = True,
     mcp_api_enabled: bool = False,
     mcp_mount_path: str = "/mcp",
-    run_migrations: bool = True,
     initialize_memory: bool = True
 ) -> FastAPI:
     """
     Create and configure the unified Hindsight API application.
 
     Args:
-        memory: MemoryEngine instance (already initialized with required parameters)
+        memory: MemoryEngine instance (already initialized with required parameters).
+                Migrations are controlled by the MemoryEngine's run_migrations parameter.
         http_api_enabled: Whether to enable HTTP REST API endpoints (default: True)
         mcp_api_enabled: Whether to enable MCP server (default: False)
         mcp_mount_path: Path to mount MCP server (default: /mcp)
-        run_migrations: Whether to run database migrations on startup (default: True)
         initialize_memory: Whether to initialize memory system on startup (default: True)
 
     Returns:
@@ -50,7 +49,6 @@ def create_app(
         from .http import create_app as create_http_app
         app = create_http_app(
             memory=memory,
-            run_migrations=run_migrations,
             initialize_memory=initialize_memory
         )
         logger.info("HTTP REST API enabled")

{hindsight_api-0.0.21 → hindsight_api-0.1.0}/hindsight_api/api/http.py

@@ -36,27 +36,13 @@ from pydantic import BaseModel, Field, ConfigDict
 from hindsight_api import MemoryEngine
 from hindsight_api.engine.memory_engine import Budget
 from hindsight_api.engine.db_utils import acquire_with_retry
+from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
 from hindsight_api.metrics import get_metrics_collector, initialize_metrics, create_metrics_collector
 
 
 logger = logging.getLogger(__name__)
 
 
-class MetadataFilter(BaseModel):
-    """Filter for metadata fields. Matches records where (key=value) OR (key not set) when match_unset=True."""
-    model_config = ConfigDict(json_schema_extra={
-        "example": {
-            "key": "source",
-            "value": "slack",
-            "match_unset": True
-        }
-    })
-
-    key: str = Field(description="Metadata key to filter on")
-    value: Optional[str] = Field(default=None, description="Value to match. If None with match_unset=True, matches any record where key is not set.")
-    match_unset: bool = Field(default=True, description="If True, also match records where this metadata key is not set")
-
-
 class EntityIncludeOptions(BaseModel):
     """Options for including entity observations in recall results."""
     max_tokens: int = Field(default=500, description="Maximum tokens for entity observations")
@@ -89,7 +75,6 @@ class RecallRequest(BaseModel):
             "max_tokens": 4096,
             "trace": True,
             "query_timestamp": "2023-05-30T23:40:00",
-            "filters": [{"key": "source", "value": "slack", "match_unset": True}],
             "include": {
                 "entities": {
                     "max_tokens": 500
@@ -104,7 +89,6 @@ class RecallRequest(BaseModel):
     max_tokens: int = 4096
     trace: bool = False
     query_timestamp: Optional[str] = Field(default=None, description="ISO format date string (e.g., '2023-05-30T23:40:00')")
-    filters: Optional[List[MetadataFilter]] = Field(default=None, description="Filter by metadata. Multiple filters are ANDed together.")
     include: IncludeOptions = Field(default_factory=IncludeOptions, description="Options for including additional data (entities are included by default)")
 
 
@@ -362,7 +346,6 @@ class ReflectRequest(BaseModel):
             "query": "What do you think about artificial intelligence?",
             "budget": "low",
             "context": "This is for a research paper on AI ethics",
-            "filters": [{"key": "source", "value": "slack", "match_unset": True}],
             "include": {
                 "facts": {}
             }
@@ -372,7 +355,6 @@ class ReflectRequest(BaseModel):
     query: str
     budget: Budget = Budget.LOW
     context: Optional[str] = None
-    filters: Optional[List[MetadataFilter]] = Field(default=None, description="Filter by metadata. Multiple filters are ANDed together.")
     include: ReflectIncludeOptions = Field(default_factory=ReflectIncludeOptions, description="Options for including additional data (disabled by default)")
 
 
@@ -439,24 +421,18 @@ class BanksResponse(BaseModel):
 
 
 class DispositionTraits(BaseModel):
-    """Disposition traits based on Big Five model."""
+    """Disposition traits that influence how memories are formed and interpreted."""
     model_config = ConfigDict(json_schema_extra={
         "example": {
-            "openness": 0.8,
-            "conscientiousness": 0.6,
-            "extraversion": 0.5,
-            "agreeableness": 0.7,
-            "neuroticism": 0.3,
-            "bias_strength": 0.7
+            "skepticism": 3,
+            "literalism": 3,
+            "empathy": 3
         }
     })
 
-    openness: float = Field(ge=0.0, le=1.0, description="Openness to experience (0-1)")
-    conscientiousness: float = Field(ge=0.0, le=1.0, description="Conscientiousness (0-1)")
-    extraversion: float = Field(ge=0.0, le=1.0, description="Extraversion (0-1)")
-    agreeableness: float = Field(ge=0.0, le=1.0, description="Agreeableness (0-1)")
-    neuroticism: float = Field(ge=0.0, le=1.0, description="Neuroticism (0-1)")
-    bias_strength: float = Field(ge=0.0, le=1.0, description="How strongly disposition influences opinions (0-1)")
+    skepticism: int = Field(ge=1, le=5, description="How skeptical vs trusting (1=trusting, 5=skeptical)")
+    literalism: int = Field(ge=1, le=5, description="How literally to interpret information (1=flexible, 5=literal)")
+    empathy: int = Field(ge=1, le=5, description="How much to consider emotional context (1=detached, 5=empathetic)")
 
 
 class BankProfileResponse(BaseModel):
@@ -466,12 +442,9 @@ class BankProfileResponse(BaseModel):
             "bank_id": "user123",
             "name": "Alice",
             "disposition": {
-                "openness": 0.8,
-                "conscientiousness": 0.6,
-                "extraversion": 0.5,
-                "agreeableness": 0.7,
-                "neuroticism": 0.3,
-                "bias_strength": 0.7
+                "skepticism": 3,
+                "literalism": 3,
+                "empathy": 3
             },
             "background": "I am a software engineer with 10 years of experience in startups"
         }
@@ -500,7 +473,7 @@ class AddBackgroundRequest(BaseModel):
     content: str = Field(description="New background information to add or merge")
     update_disposition: bool = Field(
         default=True,
-        description="If true, infer Big Five disposition traits from the merged background (default: true)"
+        description="If true, infer disposition traits from the merged background (default: true)"
     )
 
 
@@ -510,12 +483,9 @@ class BackgroundResponse(BaseModel):
         "example": {
             "background": "I was born in Texas. I am a software engineer with 10 years of experience.",
             "disposition": {
-                "openness": 0.7,
-                "conscientiousness": 0.6,
-                "extraversion": 0.5,
-                "agreeableness": 0.8,
-                "neuroticism": 0.4,
-                "bias_strength": 0.6
+                "skepticism": 3,
+                "literalism": 3,
+                "empathy": 3
             }
         }
     })
@@ -543,12 +513,9 @@ class BankListResponse(BaseModel):
                     "bank_id": "user123",
                     "name": "Alice",
                     "disposition": {
-                        "openness": 0.5,
-                        "conscientiousness": 0.5,
-                        "extraversion": 0.5,
-                        "agreeableness": 0.5,
-                        "neuroticism": 0.5,
-                        "bias_strength": 0.5
+                        "skepticism": 3,
+                        "literalism": 3,
+                        "empathy": 3
                     },
                     "background": "I am a software engineer",
                     "created_at": "2024-01-15T10:30:00Z",
@@ -567,12 +534,9 @@ class CreateBankRequest(BaseModel):
         "example": {
             "name": "Alice",
             "disposition": {
-                "openness": 0.8,
-                "conscientiousness": 0.6,
-                "extraversion": 0.5,
-                "agreeableness": 0.7,
-                "neuroticism": 0.3,
-                "bias_strength": 0.7
+                "skepticism": 3,
+                "literalism": 3,
+                "empathy": 3
             },
             "background": "I am a creative software engineer with 10 years of experience"
         }
@@ -715,13 +679,13 @@ class DeleteResponse(BaseModel):
     success: bool
 
 
-def create_app(memory: MemoryEngine, run_migrations: bool = True, initialize_memory: bool = True) -> FastAPI:
+def create_app(memory: MemoryEngine, initialize_memory: bool = True) -> FastAPI:
     """
     Create and configure the FastAPI application.
 
     Args:
-        memory: MemoryEngine instance (already initialized with required parameters)
-        run_migrations: Whether to run database migrations on startup (default: True)
+        memory: MemoryEngine instance (already initialized with required parameters).
+                Migrations are controlled by the MemoryEngine's run_migrations parameter.
         initialize_memory: Whether to initialize memory system on startup (default: True)
 
     Returns:
@@ -752,16 +716,11 @@ def create_app(memory: MemoryEngine, run_migrations: bool = True, initialize_mem
             app.state.prometheus_reader = None
             # Metrics collector is already initialized as no-op by default
 
-        # Startup: Initialize database and memory system
+        # Startup: Initialize database and memory system (migrations run inside initialize if enabled)
         if initialize_memory:
             await memory.initialize()
             logging.info("Memory system initialized")
 
-        if run_migrations:
-            from hindsight_api.migrations import run_migrations as do_migrations
-            do_migrations(memory.db_url)
-            logging.info("Database migrations applied")
-
 
 
         yield
@@ -913,17 +872,8 @@ def _register_routes(app: FastAPI):
         metrics = get_metrics_collector()
 
         try:
-            # Validate types
-            valid_fact_types = ["world", "experience", "opinion"]
-
             # Default to world, experience, opinion if not specified (exclude observation by default)
-            fact_types = request.types if request.types else ["world", "experience", "opinion"]
-            for ft in fact_types:
-                if ft not in valid_fact_types:
-                    raise HTTPException(
-                        status_code=400,
-                        detail=f"Invalid type '{ft}'. Must be one of: {', '.join(valid_fact_types)}"
-                    )
+            fact_types = request.types if request.types else list(VALID_RECALL_FACT_TYPES)
 
             # Parse query_timestamp if provided
             question_date = None
@@ -1605,7 +1555,7 @@ This operation cannot be undone.
         "/v1/default/banks/{bank_id}/profile",
         response_model=BankProfileResponse,
         summary="Update memory bank disposition",
-        description="Update bank's Big Five disposition traits and bias strength",
+        description="Update bank's disposition traits (skepticism, literalism, empathy)",
         operation_id="update_bank_disposition"
     )
     async def api_update_bank_disposition(bank_id: str,
@@ -1852,7 +1802,7 @@ This operation cannot be undone.
         "/v1/default/banks/{bank_id}/memories",
         response_model=DeleteResponse,
         summary="Clear memory bank memories",
-        description="Delete memory units for a memory bank. Optionally filter by type (world, experience, opinion) to delete only specific types. This is a destructive operation that cannot be undone. The bank profile (personality and background) will be preserved.",
+        description="Delete memory units for a memory bank. Optionally filter by type (world, experience, opinion) to delete only specific types. This is a destructive operation that cannot be undone. The bank profile (disposition and background) will be preserved.",
         operation_id="clear_bank_memories"
     )
     async def api_clear_bank_memories(bank_id: str,

{hindsight_api-0.0.21 → hindsight_api-0.1.0}/hindsight_api/api/mcp.py

@@ -8,6 +8,7 @@ from typing import Optional
 
 from fastmcp import FastMCP
 from hindsight_api import MemoryEngine
+from hindsight_api.engine.response_models import VALID_RECALL_FACT_TYPES
 
 # Configure logging from HINDSIGHT_API_LOG_LEVEL environment variable
 _log_level_str = os.environ.get("HINDSIGHT_API_LOG_LEVEL", "info").lower()
@@ -90,7 +91,7 @@ def create_mcp_server(memory: MemoryEngine) -> FastMCP:
             search_result = await memory.recall_async(
                 bank_id=bank_id,
                 query=query,
-                fact_type=["world", "experience", "opinion"],
+                fact_type=list(VALID_RECALL_FACT_TYPES),
                 budget=Budget.LOW
             )
 

{hindsight_api-0.0.21 → hindsight_api-0.1.0}/hindsight_api/cli.py

@@ -102,7 +102,6 @@ def main():
         http_api_enabled=True,
         mcp_api_enabled=True,
         mcp_mount_path="/mcp",
-        run_migrations=True,
         initialize_memory=True,
     )
 

{hindsight_api-0.0.21 → hindsight_api-0.1.0}/hindsight_api/engine/cross_encoder.py

@@ -78,7 +78,12 @@ class SentenceTransformersCrossEncoder(CrossEncoderModel):
             )
 
         logger.info(f"Loading cross-encoder model: {self.model_name}...")
-        self._model = CrossEncoder(self.model_name)
+        # Disable lazy loading (meta tensors) which causes issues with newer transformers/accelerate
+        # Setting low_cpu_mem_usage=False and device_map=None ensures tensors are fully materialized
+        self._model = CrossEncoder(
+            self.model_name,
+            model_kwargs={"low_cpu_mem_usage": False, "device_map": None},
+        )
         logger.info("Cross-encoder model loaded")
 
     def predict(self, pairs: List[Tuple[str, str]]) -> List[float]:

{hindsight_api-0.0.21 → hindsight_api-0.1.0}/hindsight_api/engine/embeddings.py

@@ -84,7 +84,12 @@ class SentenceTransformersEmbeddings(Embeddings):
             )
 
         logger.info(f"Loading embedding model: {self.model_name}...")
-        self._model = SentenceTransformer(self.model_name)
+        # Disable lazy loading (meta tensors) which causes issues with newer transformers/accelerate
+        # Setting low_cpu_mem_usage=False and device_map=None ensures tensors are fully materialized
+        self._model = SentenceTransformer(
+            self.model_name,
+            model_kwargs={"low_cpu_mem_usage": False, "device_map": None},
+        )
 
         # Validate dimension matches database schema
         model_dim = self._model.get_sentence_embedding_dimension()

{hindsight_api-0.0.21 → hindsight_api-0.1.0}/hindsight_api/engine/entity_resolver.py

@@ -126,18 +126,20 @@ class EntityResolver:
 
         # Resolve each entity using pre-fetched candidates
         entity_ids = [None] * len(entities_data)
-        entities_to_update = []  # (entity_id, unit_event_date)
-        entities_to_create = []  # (idx, entity_data)
+        entities_to_update = []  # (entity_id, event_date)
+        entities_to_create = []  # (idx, entity_data, event_date)
 
         for idx, entity_data in enumerate(entities_data):
             entity_text = entity_data['text']
             nearby_entities = entity_data.get('nearby_entities', [])
+            # Use per-entity date if available, otherwise fall back to batch-level date
+            entity_event_date = entity_data.get('event_date', unit_event_date)
 
             candidates = all_candidates.get(entity_text, [])
 
             if not candidates:
                 # Will create new entity
-                entities_to_create.append((idx, entity_data))
+                entities_to_create.append((idx, entity_data, entity_event_date))
                 continue
 
             # Score candidates
@@ -165,9 +167,9 @@ class EntityResolver:
                     score += co_entity_score * 0.3
 
                 # 3. Temporal proximity (0-0.2)
-                if last_seen:
+                if last_seen and entity_event_date:
                     # Normalize timezone awareness for comparison
-                    event_date_utc = unit_event_date if unit_event_date.tzinfo else unit_event_date.replace(tzinfo=timezone.utc)
+                    event_date_utc = entity_event_date if entity_event_date.tzinfo else entity_event_date.replace(tzinfo=timezone.utc)
                     last_seen_utc = last_seen if last_seen.tzinfo else last_seen.replace(tzinfo=timezone.utc)
                     days_diff = abs((event_date_utc - last_seen_utc).total_seconds() / 86400)
                     if days_diff < 7:
@@ -183,9 +185,9 @@ class EntityResolver:
 
             if best_score > threshold:
                 entity_ids[idx] = best_candidate
-                entities_to_update.append((best_candidate, unit_event_date))
+                entities_to_update.append((best_candidate, entity_event_date))
             else:
-                entities_to_create.append((idx, entity_data))
+                entities_to_create.append((idx, entity_data, entity_event_date))
 
         # Batch update existing entities
         if entities_to_update:
@@ -199,29 +201,54 @@ class EntityResolver:
                 entities_to_update
             )
 
-        # Create new entities using INSERT ... ON CONFLICT to handle race conditions
-        # This ensures that if two concurrent transactions try to create the same entity,
-        # only one succeeds and the other gets the existing ID
+        # Batch create new entities using COPY + INSERT for maximum speed
+        # This handles duplicates via ON CONFLICT and returns all IDs
         if entities_to_create:
-            for idx, entity_data in entities_to_create:
-                # Use INSERT ... ON CONFLICT to atomically get-or-create
-                # The unique index is on (bank_id, LOWER(canonical_name))
-                row = await conn.fetchrow(
-                    """
-                    INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count)
-                    VALUES ($1, $2, $3, $4, 1)
-                    ON CONFLICT (bank_id, LOWER(canonical_name))
-                    DO UPDATE SET
-                        mention_count = entities.mention_count + 1,
-                        last_seen = EXCLUDED.last_seen
-                    RETURNING id
-                    """,
-                    bank_id,
-                    entity_data['text'],
-                    unit_event_date,
-                    unit_event_date
-                )
-                entity_ids[idx] = row['id']
+            # Group entities by canonical name (lowercase) to handle duplicates within batch
+            # For duplicates, we only insert once and reuse the ID
+            unique_entities = {}  # lowercase_name -> (entity_data, event_date, [indices])
+            for idx, entity_data, event_date in entities_to_create:
+                name_lower = entity_data['text'].lower()
+                if name_lower not in unique_entities:
+                    unique_entities[name_lower] = (entity_data, event_date, [idx])
+                else:
+                    # Same entity appears multiple times - add index to list
+                    unique_entities[name_lower][2].append(idx)
+
+            # Batch insert unique entities and get their IDs
+            # Use a single query with unnest for speed
+            entity_names = []
+            entity_dates = []
+            indices_map = []  # Maps result index -> list of original indices
+
+            for name_lower, (entity_data, event_date, indices) in unique_entities.items():
+                entity_names.append(entity_data['text'])
+                entity_dates.append(event_date)
+                indices_map.append(indices)
+
+            # Batch INSERT ... ON CONFLICT with RETURNING
+            # This is much faster than individual inserts
+            rows = await conn.fetch(
+                """
+                INSERT INTO entities (bank_id, canonical_name, first_seen, last_seen, mention_count)
+                SELECT $1, name, event_date, event_date, 1
+                FROM unnest($2::text[], $3::timestamptz[]) AS t(name, event_date)
+                ON CONFLICT (bank_id, LOWER(canonical_name))
+                DO UPDATE SET
+                    mention_count = entities.mention_count + 1,
+                    last_seen = EXCLUDED.last_seen
+                RETURNING id
+                """,
+                bank_id,
+                entity_names,
+                entity_dates
+            )
+
+            # Map returned IDs back to original indices
+            for result_idx, row in enumerate(rows):
+                entity_id = row['id']
+                for original_idx in indices_map[result_idx]:
+                    entity_ids[original_idx] = entity_id
 
         return entity_ids