hindsight-api 0.0.21__py3-none-any.whl → 0.1.1__py3-none-any.whl

hindsight_api/engine/response_models.py

@@ -10,27 +10,28 @@ from typing import Optional, List, Dict, Any
 from pydantic import BaseModel, Field, ConfigDict
 
 
+# Valid fact types for recall operations (excludes 'observation' which is internal)
+VALID_RECALL_FACT_TYPES = frozenset(["world", "experience", "opinion"])
+
+
 class DispositionTraits(BaseModel):
     """
-    Disposition traits for a bank using the Big Five model.
+    Disposition traits for a memory bank.
 
-    All traits are scored 0.0-1.0 where higher values indicate stronger presence of the trait.
+    All traits are scored 1-5 where:
+    - skepticism: 1=trusting, 5=skeptical (how much to doubt or question information)
+    - literalism: 1=flexible interpretation, 5=literal interpretation (how strictly to interpret information)
+    - empathy: 1=detached, 5=empathetic (how much to consider emotional context)
     """
-    openness: float = Field(description="Openness to experience (0.0-1.0)")
-    conscientiousness: float = Field(description="Conscientiousness and organization (0.0-1.0)")
-    extraversion: float = Field(description="Extraversion and sociability (0.0-1.0)")
-    agreeableness: float = Field(description="Agreeableness and cooperation (0.0-1.0)")
-    neuroticism: float = Field(description="Emotional sensitivity and neuroticism (0.0-1.0)")
-    bias_strength: float = Field(description="How strongly disposition influences thinking (0.0-1.0)")
+    skepticism: int = Field(ge=1, le=5, description="How skeptical vs trusting (1=trusting, 5=skeptical)")
+    literalism: int = Field(ge=1, le=5, description="How literally to interpret information (1=flexible, 5=literal)")
+    empathy: int = Field(ge=1, le=5, description="How much to consider emotional context (1=detached, 5=empathetic)")
 
     model_config = ConfigDict(json_schema_extra={
         "example": {
-            "openness": 0.8,
-            "conscientiousness": 0.6,
-            "extraversion": 0.4,
-            "agreeableness": 0.7,
-            "neuroticism": 0.3,
-            "bias_strength": 0.5
+            "skepticism": 3,
+            "literalism": 3,
+            "empathy": 3
         }
     })
 
@@ -71,9 +72,6 @@ class MemoryFact(BaseModel):
     metadata: Optional[Dict[str, str]] = Field(None, description="User-defined metadata")
     chunk_id: Optional[str] = Field(None, description="ID of the chunk this fact was extracted from (format: bank_id_document_id_chunk_index)")
 
-    # Internal metrics (used by system but may not be exposed in API)
-    activation: Optional[float] = Field(None, description="Internal activation score")
-
 
 class ChunkInfo(BaseModel):
     """Information about a chunk."""

hindsight_api/engine/retain/bank_utils.py

@@ -13,12 +13,9 @@ from ..response_models import DispositionTraits
 logger = logging.getLogger(__name__)
 
 DEFAULT_DISPOSITION = {
-    "openness": 0.5,
-    "conscientiousness": 0.5,
-    "extraversion": 0.5,
-    "agreeableness": 0.5,
-    "neuroticism": 0.5,
-    "bias_strength": 0.5,
+    "skepticism": 3,
+    "literalism": 3,
+    "empathy": 3,
 }
 
 
@@ -32,7 +29,7 @@ class BankProfile(TypedDict):
 class BackgroundMergeResponse(BaseModel):
     """LLM response for background merge with disposition inference."""
     background: str = Field(description="Merged background in first person perspective")
-    disposition: DispositionTraits = Field(description="Inferred Big Five disposition traits")
+    disposition: DispositionTraits = Field(description="Inferred disposition traits (skepticism, literalism, empathy)")
 
 
 async def get_bank_profile(pool, bank_id: str) -> BankProfile:
@@ -92,7 +89,7 @@ async def get_bank_profile(pool, bank_id: str) -> BankProfile:
 async def update_bank_disposition(
     pool,
     bank_id: str,
-    disposition: Dict[str, float]
+    disposition: Dict[str, int]
 ) -> None:
     """
     Update bank disposition traits.
@@ -100,7 +97,7 @@ async def update_bank_disposition(
     Args:
         pool: Database connection pool
         bank_id: bank IDentifier
-        disposition: Dict with Big Five traits + bias_strength (all 0-1)
+        disposition: Dict with skepticism, literalism, empathy (all 1-5)
     """
     # Ensure bank exists first
     await get_bank_profile(pool, bank_id)
@@ -223,13 +220,10 @@ Instructions:
 3. Keep additions that don't conflict
 4. Output in FIRST PERSON ("I") perspective
 5. Be concise - keep merged background under 500 characters
-6. Infer Big Five disposition traits from the merged background:
-   - Openness: 0.0-1.0 (creativity, curiosity, openness to new ideas)
-   - Conscientiousness: 0.0-1.0 (organization, discipline, goal-directed)
-   - Extraversion: 0.0-1.0 (sociability, assertiveness, energy from others)
-   - Agreeableness: 0.0-1.0 (cooperation, empathy, consideration)
-   - Neuroticism: 0.0-1.0 (emotional sensitivity, anxiety, stress response)
-   - Bias Strength: 0.0-1.0 (how much disposition influences opinions)
+6. Infer disposition traits from the merged background (each 1-5 integer):
+   - Skepticism: 1-5 (1=trusting, takes things at face value; 5=skeptical, questions everything)
+   - Literalism: 1-5 (1=flexible interpretation, reads between lines; 5=literal, exact interpretation)
+   - Empathy: 1-5 (1=detached, focuses on facts; 5=empathetic, considers emotional context)
 
 CRITICAL: You MUST respond with ONLY a valid JSON object. No markdown, no code blocks, no explanations. Just the JSON.
 
@@ -237,22 +231,19 @@ Format:
 {{
   "background": "the merged background text in first person",
   "disposition": {{
-    "openness": 0.7,
-    "conscientiousness": 0.6,
-    "extraversion": 0.5,
-    "agreeableness": 0.8,
-    "neuroticism": 0.4,
-    "bias_strength": 0.6
+    "skepticism": 3,
+    "literalism": 3,
+    "empathy": 3
   }}
 }}
 
 Trait inference examples:
-- "creative artist" → openness: 0.8+, bias_strength: 0.6
-- "organized engineer" → conscientiousness: 0.8+, openness: 0.5-0.6
-- "startup founder" → openness: 0.8+, extraversion: 0.7+, neuroticism: 0.3-0.4
-- "risk-averse analyst" → openness: 0.3-0.4, conscientiousness: 0.8+, neuroticism: 0.6+
-- "rational and diligent" → conscientiousness: 0.7+, openness: 0.6+
-- "passionate and dramatic" → extraversion: 0.7+, neuroticism: 0.6+, openness: 0.7+"""
+- "I'm a lawyer" → skepticism: 4, literalism: 5, empathy: 2
+- "I'm a therapist" → skepticism: 2, literalism: 2, empathy: 5
+- "I'm an engineer" → skepticism: 3, literalism: 4, empathy: 3
+- "I've been burned before by trusting people" → skepticism: 5, literalism: 3, empathy: 3
+- "I try to understand what people really mean" → skepticism: 3, literalism: 2, empathy: 4
+- "I take contracts very seriously" → skepticism: 4, literalism: 5, empathy: 2"""
     else:
         prompt = f"""You are helping maintain a memory bank's background/profile.
 
@@ -282,7 +273,7 @@ Merged background:"""
                     response_format=BackgroundMergeResponse,
                     scope="bank_background",
                     temperature=0.3,
-                    max_tokens=8192
+                    max_completion_tokens=8192
                 )
                 logger.info(f"Successfully got structured response: background={parsed.background[:100]}")
 
@@ -300,7 +291,7 @@ Merged background:"""
             messages=messages,
             scope="bank_background",
             temperature=0.3,
-            max_tokens=8192
+            max_completion_tokens=8192
         )
 
         logger.info(f"LLM response for background merge (first 500 chars): {content[:500]}")
@@ -349,13 +340,12 @@ Merged background:"""
 
             # Validate disposition values
             disposition = result.get("disposition", {})
-            for key in ["openness", "conscientiousness", "extraversion",
-                       "agreeableness", "neuroticism", "bias_strength"]:
+            for key in ["skepticism", "literalism", "empathy"]:
                 if key not in disposition:
-                    disposition[key] = 0.5  # Default to neutral
+                    disposition[key] = 3  # Default to neutral
                 else:
-                    # Clamp to [0, 1]
-                    disposition[key] = max(0.0, min(1.0, float(disposition[key])))
+                    # Clamp to [1, 5] and convert to int
+                    disposition[key] = max(1, min(5, int(disposition[key])))
 
             result["disposition"] = disposition
 

hindsight_api/engine/retain/entity_processing.py

@@ -7,7 +7,7 @@ import logging
 from typing import List, Tuple, Dict, Any
 from uuid import UUID
 
-from .types import ProcessedFact, EntityRef
+from .types import ProcessedFact, EntityRef, EntityLink
 from . import link_utils
 
 logger = logging.getLogger(__name__)
@@ -20,7 +20,7 @@ async def process_entities_batch(
     unit_ids: List[str],
     facts: List[ProcessedFact],
     log_buffer: List[str] = None
-) -> List[Tuple[str, str, float]]:
+) -> List[EntityLink]:
     """
     Process entities for all facts and create entity links.
 
@@ -39,7 +39,7 @@ async def process_entities_batch(
         log_buffer: Optional buffer for detailed logging
 
     Returns:
-        List of entity link tuples: (unit_id, entity_id, confidence)
+        List of EntityLink objects for batch insertion
     """
     if not unit_ids or not facts:
         return []
@@ -75,14 +75,14 @@ async def process_entities_batch(
 
 async def insert_entity_links_batch(
     conn,
-    entity_links: List[Tuple[str, str, float]]
+    entity_links: List[EntityLink]
 ) -> None:
     """
     Insert entity links in batch.
 
     Args:
         conn: Database connection
-        entity_links: List of (unit_id, entity_id, confidence) tuples
+        entity_links: List of EntityLink objects
     """
     if not entity_links:
         return

hindsight_api/engine/retain/fact_extraction.py

@@ -167,10 +167,10 @@ class ExtractedFact(BaseModel):
         description="'world' = about the user/others (background, experiences). 'assistant' = experience with the assistant."
     )
 
-    # Entities - extracted from 'who' field
+    # Entities - extracted from fact content
     entities: Optional[List[Entity]] = Field(
         default=None,
-        description="Named entities from 'who': people names, organizations, places. NOT generic relations."
+        description="Named entities, objects, AND abstract concepts from the fact. Include: people names, organizations, places, significant objects (e.g., 'coffee maker', 'car'), AND abstract concepts/themes (e.g., 'friendship', 'career growth', 'loss', 'celebration'). Extract anything that could help link related facts together."
     )
     causal_relations: Optional[List[CausalRelation]] = Field(
         default=None,
@@ -325,7 +325,7 @@ async def _extract_facts_from_chunk(
     Note: event_date parameter is kept for backward compatibility but not used in prompt.
     The LLM extracts temporal information from the context string instead.
     """
-    agent_context = f"\n- Your name: {agent_name}" if agent_name else ""
+    memory_bank_context = f"\n- Your name: {agent_name}" if agent_name and extract_opinions else ""
 
     # Determine which fact types to extract based on the flag
     # Note: We use "assistant" in the prompt but convert to "bank" for storage
@@ -339,7 +339,7 @@ async def _extract_facts_from_chunk(
 
 {fact_types_instruction}
 
-Context: {context if context else 'none'}{agent_context}
+
 
 ══════════════════════════════════════════════════════════════════════════
 FACT FORMAT - ALL FIVE DIMENSIONS REQUIRED - MAXIMUM VERBOSITY
@@ -382,13 +382,42 @@ WRONG output:
 - where: (missing) ← WRONG - include the location!
 
 ══════════════════════════════════════════════════════════════════════════
-TEMPORAL HANDLING
+FACT_KIND CLASSIFICATION (CRITICAL FOR TEMPORAL HANDLING)
 ══════════════════════════════════════════════════════════════════════════
 
-For EVENTS (fact_kind="event"):
-- Convert relative dates → absolute WITH DAY OF WEEK: "yesterday" on Saturday March 15 → "Friday, March 14, 2024"
+⚠️ MUST set fact_kind correctly - this determines whether occurred_start/end are set!
+
+fact_kind="event" - USE FOR:
+- Actions that happened at a specific time: "went to", "attended", "visited", "bought", "made"
+- Past events: "yesterday I...", "last week...", "in March 2020..."
+- Future plans with dates: "will go to", "scheduled for"
+- Examples: "I went to a pottery workshop" → event
+           "Alice visited Paris in February" → event
+           "I bought a new car yesterday" → event
+           "The user graduated from MIT in March 2020" → event
+
+fact_kind="conversation" - USE FOR:
+- Ongoing states: "works as", "lives in", "is married to"
+- Preferences: "loves", "prefers", "enjoys"
+- Traits/abilities: "speaks fluent French", "knows Python"
+- Examples: "I love Italian food" → conversation
+           "Alice works at Google" → conversation
+           "I prefer outdoor dining" → conversation
+
+══════════════════════════════════════════════════════════════════════════
+TEMPORAL HANDLING (CRITICAL - USE EVENT DATE AS REFERENCE)
+══════════════════════════════════════════════════════════════════════════
+
+⚠️ IMPORTANT: Use the "Event Date" provided in the input as your reference point!
+All relative dates ("yesterday", "last week", "recently") must be resolved relative to the Event Date, NOT today's date.
+
+For EVENTS (fact_kind="event") - MUST SET BOTH occurred_start AND occurred_end:
+- Convert relative dates → absolute using Event Date as reference
+- If Event Date is "Saturday, March 15, 2020", then "yesterday" = Friday, March 14, 2020
+- Dates mentioned in text (e.g., "in March 2020") should use THAT year, not current year
 - Always include the day name (Monday, Tuesday, etc.) in the 'when' field
-- Set occurred_start/occurred_end to WHEN IT HAPPENED (not when mentioned)
+- Set occurred_start AND occurred_end to WHEN IT HAPPENED (not when mentioned)
+- For single-day/point events: set occurred_end = occurred_start (same timestamp)
 
 For CONVERSATIONS (fact_kind="conversation"):
 - General info, preferences, ongoing states → NO occurred dates
@@ -415,20 +444,32 @@ Example: "I love Italian food and prefer outdoor dining"
 → Fact 2: what="User prefers outdoor dining", who="user", why="This is a dining preference", entities=["user"]
 
 ══════════════════════════════════════════════════════════════════════════
-ENTITIES - INCLUDE "user" (CRITICAL)
+ENTITIES - INCLUDE PEOPLE, PLACES, OBJECTS, AND CONCEPTS (CRITICAL)
 ══════════════════════════════════════════════════════════════════════════
 
-When a fact is ABOUT the user (their preferences, plans, experiences), ALWAYS include "user" in entities!
-
-✅ CORRECT: entities=["user"] for "User loves coffee"
-✅ CORRECT: entities=["user", "Emily"] for "User attended Emily's wedding"
-❌ WRONG: entities=[] for facts about the user
+Extract entities that help link related facts together. Include:
+1. "user" - when the fact is about the user
+2. People names - Emily, Dr. Smith, etc.
+3. Organizations/Places - IKEA, Goodwill, New York, etc.
+4. Specific objects - coffee maker, toaster, car, laptop, kitchen, etc.
+5. Abstract concepts - themes, values, emotions, or ideas that capture the essence of the fact:
+   - "friendship" for facts about friends helping each other, bonding, loyalty
+   - "career growth" for facts about promotions, learning new skills, job changes
+   - "loss" or "grief" for facts about death, endings, saying goodbye
+   - "celebration" for facts about parties, achievements, milestones
+   - "trust" or "betrayal" for facts involving those themes
+
+✅ CORRECT: entities=["user", "coffee maker", "Goodwill", "kitchen"] for "User donated their coffee maker to Goodwill"
+✅ CORRECT: entities=["user", "Emily", "friendship"] for "Emily helped user move to a new apartment"
+✅ CORRECT: entities=["user", "promotion", "career growth"] for "User got promoted to senior engineer"
+✅ CORRECT: entities=["user", "grandmother", "loss", "grief"] for "User's grandmother passed away last week"
+❌ WRONG: entities=["user", "Emily"] only - missing the "friendship" concept that links to other friendship facts!
 
 ══════════════════════════════════════════════════════════════════════════
 EXAMPLES
 ══════════════════════════════════════════════════════════════════════════
 
-Example 1 - World Facts (Context: June 10, 2024):
+Example 1 - World Facts (Event Date: Tuesday, June 10, 2024):
 Input: "I'm planning my wedding and want a small outdoor ceremony. I just got back from my college roommate Emily's wedding - she married Sarah at a rooftop garden, it was so romantic!"
 
 Output facts:
@@ -438,22 +479,23 @@ Output facts:
    - who: "user"
    - why: "User prefers intimate outdoor settings"
    - fact_type: "world", fact_kind: "conversation"
-   - entities: ["user"]
+   - entities: ["user", "wedding", "outdoor ceremony"]
 
 2. User planning wedding
    - what: "User is planning their own wedding"
    - who: "user"
    - why: "Inspired by Emily's ceremony"
    - fact_type: "world", fact_kind: "conversation"
-   - entities: ["user"]
+   - entities: ["user", "wedding"]
 
-3. Emily's wedding (THE EVENT)
+3. Emily's wedding (THE EVENT - note occurred_start AND occurred_end both set)
    - what: "Emily got married to Sarah at a rooftop garden ceremony in the city"
    - who: "Emily (user's college roommate), Sarah (Emily's partner)"
    - why: "User found it romantic and beautiful"
    - fact_type: "world", fact_kind: "event"
-   - occurred_start: "2024-06-09T00:00:00Z" (recently, user "just got back")
-   - entities: ["user", "Emily", "Sarah"]
+   - occurred_start: "2024-06-09T00:00:00Z" (recently, user "just got back" - relative to Event Date June 10, 2024)
+   - occurred_end: "2024-06-09T23:59:59Z" (same day - point event)
+   - entities: ["user", "Emily", "Sarah", "wedding", "rooftop garden"]
 
 Example 2 - Assistant Facts (Context: March 5, 2024):
 Input: "User: My API is really slow when we have 1000+ concurrent users. What can I do?
@@ -465,7 +507,23 @@ Output fact:
    - who: "user, assistant"
    - why: "User asked how to fix slow API performance with 1000+ concurrent users, expected 70-80% reduction in database load"
    - fact_type: "assistant", fact_kind: "conversation"
-   - entities: ["user"]
+   - entities: ["user", "API", "Redis"]
+
+Example 3 - Kitchen Items with Concept Inference (Event Date: Thursday, May 30, 2024):
+Input: "I finally donated my old coffee maker to Goodwill. I upgraded to that new espresso machine last month and the old one was just taking up counter space."
+
+Output fact:
+   - what: "User donated their old coffee maker to Goodwill after upgrading to a new espresso machine"
+   - when: "Thursday, May 30, 2024"
+   - who: "user"
+   - why: "The old coffee maker was taking up counter space after the upgrade"
+   - fact_type: "world", fact_kind: "event"
+   - occurred_start: "2024-05-30T00:00:00Z" (uses Event Date year)
+   - occurred_end: "2024-05-30T23:59:59Z" (same day - point event)
+   - entities: ["user", "coffee maker", "Goodwill", "espresso machine", "kitchen"]
+
+Note: "kitchen" is inferred as a concept because coffee makers and espresso machines are kitchen appliances.
+This links the fact to other kitchen-related facts (toaster, faucet, kitchen mat, etc.) via the shared "kitchen" entity.
 
 Note how the "why" field captures the FULL STORY: what the user asked AND what outcome was expected!
 
@@ -496,6 +554,7 @@ WHAT TO EXTRACT vs SKIP
     # Format event_date with day of week for better temporal reasoning
     event_date_formatted = event_date.strftime('%A, %B %d, %Y')  # e.g., "Monday, June 10, 2024"
     user_message = f"""Extract facts from the following text chunk.
+{memory_bank_context}
 
 Chunk: {chunk_index + 1}/{total_chunks}
 Event Date: {event_date_formatted} ({event_date.isoformat()})
@@ -520,7 +579,7 @@ Text:
                 response_format=FactExtractionResponse,
                 scope="memory_extract_facts",
                 temperature=0.1,
-                max_tokens=65000,
+                max_completion_tokens=65000,
                 skip_validation=True,  # Get raw JSON, we'll validate leniently
             )
 
@@ -628,8 +687,11 @@ Text:
                     occurred_end = get_value('occurred_end')
                     if occurred_start:
                         fact_data['occurred_start'] = occurred_start
-                    if occurred_end:
-                        fact_data['occurred_end'] = occurred_end
+                        # For point events: if occurred_end not set, default to occurred_start
+                        if occurred_end:
+                            fact_data['occurred_end'] = occurred_end
+                        else:
+                            fact_data['occurred_end'] = occurred_start
 
                 # Add entities if present (validate as Entity objects)
                 # LLM sometimes returns strings instead of {"text": "..."} format

hindsight_api/engine/retain/fact_storage.py

@@ -118,7 +118,7 @@ async def ensure_bank_exists(conn, bank_id: str) -> None:
         SET updated_at = NOW()
         """,
         bank_id,
-        '{"openness": 0.5, "conscientiousness": 0.5, "extraversion": 0.5, "agreeableness": 0.5, "neuroticism": 0.5, "bias_strength": 0.5}',
+        '{"skepticism": 3, "literalism": 3, "empathy": 3}',
         ""
     )
 

hindsight_api/engine/retain/link_creation.py

@@ -16,7 +16,7 @@ async def create_temporal_links_batch(
     conn,
     bank_id: str,
     unit_ids: List[str]
-) -> None:
+) -> int:
     """
     Create temporal links between facts.
 
@@ -26,11 +26,14 @@ async def create_temporal_links_batch(
         conn: Database connection
         bank_id: Bank identifier
         unit_ids: List of unit IDs to create links for
+
+    Returns:
+        Number of temporal links created
     """
     if not unit_ids:
-        return
+        return 0
 
-    await link_utils.create_temporal_links_batch_per_fact(
+    return await link_utils.create_temporal_links_batch_per_fact(
         conn,
         bank_id,
         unit_ids,
@@ -43,7 +46,7 @@ async def create_semantic_links_batch(
     bank_id: str,
     unit_ids: List[str],
     embeddings: List[List[float]]
-) -> None:
+) -> int:
     """
     Create semantic links between facts.
 
@@ -54,14 +57,17 @@ async def create_semantic_links_batch(
         bank_id: Bank identifier
         unit_ids: List of unit IDs to create links for
         embeddings: List of embedding vectors (same length as unit_ids)
+
+    Returns:
+        Number of semantic links created
     """
     if not unit_ids or not embeddings:
-        return
+        return 0
 
     if len(unit_ids) != len(embeddings):
         raise ValueError(f"Mismatch between unit_ids ({len(unit_ids)}) and embeddings ({len(embeddings)})")
 
-    await link_utils.create_semantic_links_batch(
+    return await link_utils.create_semantic_links_batch(
         conn,
         bank_id,
         unit_ids,