hindsight-api 0.0.14__py3-none-any.whl → 0.0.15__py3-none-any.whl

hindsight_api/engine/retain/fact_extraction.py

@@ -16,6 +16,24 @@ from pydantic import BaseModel, Field, field_validator, ConfigDict
 from ..llm_wrapper import OutputTooLongError, LLMConfig
 
 
+def _sanitize_text(text: str) -> str:
+    """
+    Sanitize text by removing invalid Unicode surrogate characters.
+
+    Surrogate characters (U+D800 to U+DFFF) are used in UTF-16 encoding
+    but cannot be encoded in UTF-8. They can appear in Python strings
+    from improperly decoded data (e.g., from JavaScript or broken files).
+
+    This function removes unpaired surrogates to prevent UnicodeEncodeError
+    when the text is sent to the LLM API.
+    """
+    if not text:
+        return text
+    # Remove surrogate characters (U+D800 to U+DFFF) using regex
+    # These are invalid in UTF-8 and cause encoding errors
+    return re.sub(r'[\ud800-\udfff]', '', text)
+
+
 class Entity(BaseModel):
     """An entity extracted from text."""
     text: str = Field(
@@ -470,6 +488,10 @@ WHAT TO EXTRACT vs SKIP
     max_retries = 2
     last_error = None
 
+    # Sanitize input text to prevent Unicode encoding errors (e.g., unpaired surrogates)
+    sanitized_chunk = _sanitize_text(chunk)
+    sanitized_context = _sanitize_text(context) if context else 'none'
+
     # Build user message with metadata and chunk content in a clear format
     # Format event_date with day of week for better temporal reasoning
     event_date_formatted = event_date.strftime('%A, %B %d, %Y')  # e.g., "Monday, June 10, 2024"
@@ -477,10 +499,10 @@ WHAT TO EXTRACT vs SKIP
 
 Chunk: {chunk_index + 1}/{total_chunks}
 Event Date: {event_date_formatted} ({event_date.isoformat()})
-Context: {context if context else 'none'}
+Context: {sanitized_context}
 
 Text:
-{chunk}"""
+{sanitized_chunk}"""
 
     for attempt in range(max_retries):
         try:

{hindsight_api-0.0.14.dist-info → hindsight_api-0.0.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hindsight-api
-Version: 0.0.14
+Version: 0.0.15
 Summary: Temporal + Semantic + Entity Memory System for AI agents using PostgreSQL
 Requires-Python: >=3.11
 Requires-Dist: alembic>=1.17.1

{hindsight_api-0.0.14.dist-info → hindsight_api-0.0.15.dist-info}/RECORD

@@ -24,7 +24,7 @@ hindsight_api/engine/retain/deduplication.py,sha256=9YXgVI_m1Mtz5Cv46ZceCEs0GwpL
 hindsight_api/engine/retain/embedding_processing.py,sha256=cHTt3rPvDCWBWVPfSeg6bwH8HoXYGmP4bvS21boNONI,1734
 hindsight_api/engine/retain/embedding_utils.py,sha256=Q24h_iw6pRAW2vDWPvauWY1o3bXLzW3eWvSxDALDiE0,1588
 hindsight_api/engine/retain/entity_processing.py,sha256=meHOjsFzdvh1tbe6YlTofhcUs2Y6TcAN3S-0EKOvFP0,2705
-hindsight_api/engine/retain/fact_extraction.py,sha256=D7nnDn7U0UhsAwbo9qahPSpGxiRP-L5tdHT1JAIKM44,45254
+hindsight_api/engine/retain/fact_extraction.py,sha256=vOIlag9rJ8_8Q-TfOhMY88PeJpUyFIp0i7vdEyzbJLY,46125
 hindsight_api/engine/retain/fact_storage.py,sha256=gRRQf_FCLsj5lUvdlOaxJsS5JosM6IhO_pik8Ur8VFg,5717
 hindsight_api/engine/retain/link_creation.py,sha256=XJx7U3HboJLHtGgt_tHGsCa58lGo2ZyywzMNosrY9Xc,3154
 hindsight_api/engine/retain/link_utils.py,sha256=PAXalIhAPZGcJv8EugcpwNgoWZ2D_ciVU3brHL-m090,26226
@@ -43,6 +43,6 @@ hindsight_api/engine/search/tracer.py,sha256=mcM9qZpj3YFudrBCESwc6YKNAiWIMx1lScX
 hindsight_api/engine/search/types.py,sha256=qIeHW_gT7f291vteTZXygAM8oAaPp2dq6uEdvOyOwzs,5488
 hindsight_api/web/__init__.py,sha256=WABqyqiAVFJJWOhKCytkj5Vcb61eAsRib3Ek7IMX6_U,378
 hindsight_api/web/server.py,sha256=oPNJ_z4DO38MdK7Juyh2LdH0ipZ_BQF48cUM-4B_Uw0,5379
-hindsight_api-0.0.14.dist-info/METADATA,sha256=FFYSGyrGsxR_vYmM11h2AK4pqC8MdbOjjV46yJ0mlL8,1496
-hindsight_api-0.0.14.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-hindsight_api-0.0.14.dist-info/RECORD,,
+hindsight_api-0.0.15.dist-info/METADATA,sha256=bXvuwqAtqB2mCaDUCkiJ8t5tw085gMLWDLsGhFHF9Bo,1496
+hindsight_api-0.0.15.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+hindsight_api-0.0.15.dist-info/RECORD,,