memmachine-server 0.3.2__tar.gz → 0.3.3__tar.gz

{memmachine_server-0.3.2/src/memmachine_server.egg-info → memmachine_server-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: memmachine-server
-Version: 0.3.2
+Version: 0.3.3
 Summary: MemMachine Server - The complete MemMachine memory system server with episodic and profile memory
 Author-email: MemMachine <noreply@memmachine.ai>
 License-Expression: Apache-2.0

memmachine_server-0.3.3/server_tests/memmachine_server/common/embedder/test_openai_embedder.py

@@ -0,0 +1,126 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import openai
+import pytest
+
+from memmachine_server.common.data_types import ExternalServiceAPIError
+from memmachine_server.common.embedder.openai_embedder import (
+    OpenAIEmbedder,
+    OpenAIEmbedderParams,
+)
+
+
+@pytest.fixture(
+    params=[
+        ["Are tomatoes fruits?", "Tomatoes are red."],
+        ["Are tomatoes fruits?", "Tomatoes are red.", ""],
+        ["."],
+        [" "],
+        [""],
+        [],
+    ],
+)
+def inputs(request):
+    return request.param
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_ingest_embed(openai_embedder, inputs):
+    embeddings = await openai_embedder.ingest_embed(inputs)
+    assert isinstance(embeddings, list)
+    assert len(embeddings) == len(inputs)
+    assert all(len(embedding) == openai_embedder.dimensions for embedding in embeddings)
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_search_embed(openai_embedder, inputs):
+    embeddings = await openai_embedder.search_embed(inputs)
+    assert isinstance(embeddings, list)
+    assert len(embeddings) == len(inputs)
+    assert all(len(embedding) == openai_embedder.dimensions for embedding in embeddings)
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_large_input(openai_embedder):
+    input_text = "👩‍💻" * 10000
+
+    assert len(await openai_embedder.ingest_embed([input_text])) == 1
+    assert len(await openai_embedder.search_embed([input_text])) == 1
+
+
+@pytest.mark.integration
+@pytest.mark.asyncio
+async def test_many_inputs(openai_embedder):
+    input_texts = ["Hello, world!"] * 10000
+    assert len(await openai_embedder.ingest_embed(input_texts)) == 10000
+    assert len(await openai_embedder.search_embed(input_texts)) == 10000
+
+
+@pytest.mark.asyncio
+@patch("asyncio.sleep", new_callable=AsyncMock)
+async def test_embed_retry_on_internal_server_error(mock_sleep):
+    """Test retry logic on InternalServerError."""
+    mock_client = AsyncMock(spec=openai.AsyncOpenAI)
+
+    mock_embedding = MagicMock()
+    mock_embedding.embedding = [0.1] * 256
+
+    mock_response = MagicMock()
+    mock_response.data = [mock_embedding]
+    mock_response.usage.prompt_tokens = 10
+    mock_response.usage.total_tokens = 10
+
+    mock_client.embeddings.create = AsyncMock(
+        side_effect=[
+            openai.InternalServerError(
+                "internal server error", response=MagicMock(), body=None
+            ),
+            mock_response,
+        ]
+    )
+
+    embedder = OpenAIEmbedder(
+        OpenAIEmbedderParams(
+            client=mock_client,
+            model="test-model",
+            dimensions=256,
+        )
+    )
+
+    result = await embedder.ingest_embed(["test input"], max_attempts=2)
+
+    assert len(result) == 1
+    assert len(result[0]) == 256
+    assert mock_client.embeddings.create.call_count == 2
+    mock_sleep.assert_awaited_once_with(1)
+
+
+@pytest.mark.asyncio
+@patch("asyncio.sleep", new_callable=AsyncMock)
+async def test_embed_fail_after_max_retries_on_internal_server_error(mock_sleep):
+    """Test that ExternalServiceAPIError is raised after max_attempts on InternalServerError."""
+    mock_client = AsyncMock(spec=openai.AsyncOpenAI)
+    mock_client.embeddings.create = AsyncMock(
+        side_effect=openai.InternalServerError(
+            "internal server error", response=MagicMock(), body=None
+        ),
+    )
+
+    embedder = OpenAIEmbedder(
+        OpenAIEmbedderParams(
+            client=mock_client,
+            model="test-model",
+            dimensions=256,
+        )
+    )
+
+    with pytest.raises(ExternalServiceAPIError, match=r"max attempts"):
+        await embedder.ingest_embed(["test input"], max_attempts=3)
+
+    assert mock_client.embeddings.create.call_count == 3
+    assert mock_sleep.call_count == 2
+    mock_sleep.assert_any_await(1)
+    mock_sleep.assert_any_await(2)

{memmachine_server-0.3.2 → memmachine_server-0.3.3}/server_tests/memmachine_server/common/language_model/test_openai_chat_completions_language_model.py

@@ -397,6 +397,35 @@ async def test_generate_response_fail_after_max_retries(
     mock_sleep.assert_any_await(2)
 
 
+@pytest.mark.asyncio
+@patch("asyncio.sleep", new_callable=AsyncMock)
+async def test_generate_response_retry_on_internal_server_error(
+    mock_sleep,
+    mock_async_openai,
+    minimal_config,
+):
+    """Test retry logic on InternalServerError."""
+    mock_response = MagicMock()
+    mock_response.choices[0].message.content = "Success after retry"
+    mock_response.choices[0].message.tool_calls = None
+    mock_response.usage = None
+
+    mock_client = mock_async_openai.return_value
+    mock_client.chat.completions.create.side_effect = [
+        openai.InternalServerError(
+            "internal server error", response=MagicMock(), body=None
+        ),
+        mock_response,
+    ]
+
+    lm = OpenAIChatCompletionsLanguageModel(minimal_config)
+    content, _ = await lm.generate_response(max_attempts=2)
+
+    assert content == "Success after retry"
+    assert mock_client.chat.completions.create.call_count == 2
+    mock_sleep.assert_awaited_once_with(1)
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "exception",

{memmachine_server-0.3.2 → memmachine_server-0.3.3}/server_tests/memmachine_server/common/language_model/test_openai_responses_language_model.py

@@ -353,6 +353,35 @@ async def test_generate_response_fail_after_max_retries(
     mock_sleep.assert_any_await(2)
 
 
+@pytest.mark.asyncio
+@patch("asyncio.sleep", new_callable=AsyncMock)
+async def test_generate_response_retry_on_internal_server_error(
+    mock_sleep,
+    mock_async_openai,
+    minimal_config,
+):
+    """Test retry logic on InternalServerError."""
+    mock_response = MagicMock()
+    mock_response.output_text = "Success after retry"
+    mock_response.output = None
+    mock_response.usage = None
+
+    mock_client = mock_async_openai.return_value
+    mock_client.responses.create.side_effect = [
+        openai.InternalServerError(
+            "internal server error", response=MagicMock(), body=None
+        ),
+        mock_response,
+    ]
+
+    lm = OpenAIResponsesLanguageModel(minimal_config)
+    content, _ = await lm.generate_response(max_attempts=2)
+
+    assert content == "Success after retry"
+    assert mock_client.responses.create.call_count == 2
+    mock_sleep.assert_awaited_once_with(1)
+
+
 @pytest.mark.asyncio
 @pytest.mark.asyncio
 @pytest.mark.parametrize(

{memmachine_server-0.3.2 → memmachine_server-0.3.3}/server_tests/memmachine_server/episodic_memory/short_term_memory/test_short_term_memory.py

@@ -347,6 +347,78 @@ class TestSessionMemoryPublicAPI:
         # context window limit causes recursive splitting into 4 LLM calls
         assert mock_model.call_count == 4
 
+    @pytest.mark.asyncio
+    async def test_summary_exceed_context_window_single_episode(
+        self, mock_data_manager
+    ):
+        class ChunkingLanguageModel(LanguageModel):
+            def __init__(self) -> None:
+                self.call_count = 0
+
+            async def generate_response(
+                self,
+                system_prompt: str | None = None,
+                user_prompt: str | None = None,
+                tools: list[dict[str, Any]] | None = None,
+                tool_choice: str | dict[str, str] | None = None,
+                max_attempts: int = 1,
+            ) -> tuple[str, Any]:
+                prompt = user_prompt or ""
+                if len(prompt) > 10000:
+                    raise ValueError("User prompt exceeds context window")
+                self.call_count += 1
+                match = re.search(r"summary:(\d+)(?: \d+)?$", prompt)
+                prev_summary = int(match.group(1)) if match else 0
+                messages = re.findall(r'"([^"]*)"', prompt)
+                return (
+                    f"summary:{prev_summary + sum(len(message) for message in messages)}",
+                    "",
+                )
+
+            async def generate_response_with_token_usage(
+                self,
+                system_prompt: str | None = None,
+                user_prompt: str | None = None,
+                tools: list[dict[str, Any]] | None = None,
+                tool_choice: str | dict[str, str] | None = None,
+                max_attempts: int = 1,
+            ) -> tuple[str, Any, int, int]:
+                response, tool_output = await self.generate_response(
+                    system_prompt=system_prompt,
+                    user_prompt=user_prompt,
+                    tools=tools,
+                    tool_choice=tool_choice,
+                    max_attempts=max_attempts,
+                )
+                return response, tool_output, 0, 0
+
+            async def generate_parsed_response(
+                self,
+                output_format: type[T],
+                system_prompt: str | None = None,
+                user_prompt: str | None = None,
+                max_attempts: int = 1,
+            ) -> T:
+                return cast(T, "summary")
+
+        model = ChunkingLanguageModel()
+        params = ShortTermMemoryConsolidator.Params(
+            summary_user_prompt="User prompt: {episodes} {summary} {max_length}",
+            summary_system_prompt="System Prompt",
+            max_summary_length_words=100,
+            session_key="test_session",
+            model=model,
+            data_manager=mock_data_manager,
+        )
+        consolidator = ShortTermMemoryConsolidator(params)
+
+        oversized_content = "a" * 12000
+        episode = create_test_episode(content=oversized_content)
+        summary = await consolidator._create_summary("", [episode])
+
+        assert summary == "summary:12000"
+        assert model.call_count > 1
+
     @pytest.mark.asyncio
     async def test_summary_catch_up(self, mock_model, mock_data_manager):
         params = ShortTermMemoryConsolidator.Params(

{memmachine_server-0.3.2 → memmachine_server-0.3.3}/server_tests/memmachine_server/semantic_memory/mock_semantic_memory_objects.py

@@ -182,6 +182,9 @@ class MockSemanticStorage(SemanticStorage):
     ) -> list[SetIdT]:
         raise NotImplementedError
 
+    async def purge_ingested_rows(self, set_ids: list[SetIdT]) -> int:
+        raise NotImplementedError
+
     async def get_set_ids_starts_with(self, prefix: str) -> list[SetIdT]:
         raise NotImplementedError
 

{memmachine_server-0.3.2 → memmachine_server-0.3.3}/server_tests/memmachine_server/semantic_memory/storage/in_memory_semantic_storage.py

@@ -339,6 +339,34 @@ class InMemorySemanticStorage(SemanticStorage):
 
             return set_ids
 
+    async def purge_ingested_rows(self, set_ids: list[SetIdT]) -> int:
+        if not set_ids:
+            return 0
+        deleted = 0
+        async with self._lock:
+            for set_id in set_ids:
+                history_map = self._set_history_map.get(set_id)
+                if history_map is None:
+                    continue
+                # Skip sets that still have pending (uningested) messages
+                # to preserve the duplicate guard for in-flight work.
+                has_pending = any(not v for v in history_map.values())
+                if has_pending:
+                    continue
+                ingested_ids = list(history_map.keys())
+                for h_id in ingested_ids:
+                    del history_map[h_id]
+                    self._history_created_at.pop((set_id, h_id), None)
+                    h_sets = self._history_to_sets.get(h_id)
+                    if h_sets is not None:
+                        h_sets.pop(set_id, None)
+                        if not h_sets:
+                            self._history_to_sets.pop(h_id, None)
+                    deleted += 1
+                if not history_map:
+                    self._set_history_map.pop(set_id, None)
+        return deleted
+
     async def get_set_ids_starts_with(self, prefix: str) -> list[SetIdT]:
         async with self._lock:
             set_ids = {

{memmachine_server-0.3.2 → memmachine_server-0.3.3}/server_tests/memmachine_server/semantic_memory/storage/test_semantic_storage.py

@@ -1097,6 +1097,98 @@ async def test_get_set_ids_with_older_than_and_min_uningested(
     assert set_ids == ["busy_user"]
 
 
+@pytest.mark.asyncio
+async def test_purge_ingested_rows_fully_ingested(
+    semantic_storage: SemanticStorage,
+    episode_storage,
+):
+    """Purge deletes all rows for a set_id when every message is ingested."""
+    history_ids = [
+        await _add_episode(episode_storage, content=f"purge-msg-{idx}")
+        for idx in range(3)
+    ]
+
+    for h_id in history_ids:
+        await semantic_storage.add_history_to_set(set_id="purge_user", history_id=h_id)
+
+    # Mark all 3 as ingested
+    await semantic_storage.mark_messages_ingested(
+        set_id="purge_user",
+        history_ids=history_ids,
+    )
+
+    deleted = await semantic_storage.purge_ingested_rows(["purge_user"])
+    assert deleted == 3
+
+    total = await semantic_storage.get_history_messages_count(
+        set_ids=["purge_user"],
+    )
+    assert total == 0
+
+
+@pytest.mark.asyncio
+async def test_purge_ingested_rows_skips_partially_ingested(
+    semantic_storage: SemanticStorage,
+    episode_storage,
+):
+    """Purge preserves rows when a set still has uningested messages,
+    keeping the (set_id, history_id) duplicate guard intact."""
+    history_ids = [
+        await _add_episode(episode_storage, content=f"partial-msg-{idx}")
+        for idx in range(3)
+    ]
+
+    for h_id in history_ids:
+        await semantic_storage.add_history_to_set(
+            set_id="partial_user", history_id=h_id
+        )
+
+    # Mark 2 of 3 as ingested — 1 still pending
+    await semantic_storage.mark_messages_ingested(
+        set_id="partial_user",
+        history_ids=history_ids[:2],
+    )
+
+    # Purge should skip this set entirely
+    deleted = await semantic_storage.purge_ingested_rows(["partial_user"])
+    assert deleted == 0
+
+    # All 3 rows should remain
+    total = await semantic_storage.get_history_messages_count(
+        set_ids=["partial_user"],
+    )
+    assert total == 3
+
+
+@pytest.mark.asyncio
+async def test_purge_ingested_rows_empty_list(
+    semantic_storage: SemanticStorage,
+    episode_storage,
+):
+    history_ids = [
+        await _add_episode(episode_storage, content=f"keep-msg-{idx}")
+        for idx in range(2)
+    ]
+
+    for h_id in history_ids:
+        await semantic_storage.add_history_to_set(set_id="keep_user", history_id=h_id)
+
+    await semantic_storage.mark_messages_ingested(
+        set_id="keep_user",
+        history_ids=history_ids[:1],
+    )
+
+    # Purge with empty list should be a no-op
+    deleted = await semantic_storage.purge_ingested_rows([])
+    assert deleted == 0
+
+    # Data should be unchanged
+    total = await semantic_storage.get_history_messages_count(
+        set_ids=["keep_user"],
+    )
+    assert total == 2
+
+
 @pytest.mark.asyncio
 async def test_filter_features_by_created_at_date(
     semantic_storage: SemanticStorage,
@@ -1257,6 +1349,24 @@ async def test_filter_features_by_created_at_range(
     assert results[0].value == "note2"
 
 
+@pytest.mark.asyncio
+async def test_add_feature_with_null_bytes_does_not_crash(
+    semantic_storage: SemanticStorage,
+):
+    """Regression test for issue #1147: null bytes in feature values must not crash."""
+    feature_id = await semantic_storage.add_feature(
+        set_id="user1",
+        category_name="profile",
+        feature="cultural\x00focus",
+        value="D\x00a de Los Muertos",
+        tag="Ins\x00ights",
+        embedding=np.array([1.0] * 1536, dtype=float),
+    )
+
+    stored = await semantic_storage.get_feature(feature_id)
+    assert stored is not None
+
+
 @pytest.mark.asyncio
 async def test_filter_equality(semantic_storage: SemanticStorage):
     feature_ids: list[FeatureIdT] = [

memmachine_server-0.3.3/server_tests/memmachine_server/semantic_memory/storage/test_text_sanitizer.py

@@ -0,0 +1,48 @@
+"""Tests for the PostgreSQL text sanitizer utility."""
+
+import logging
+
+from memmachine_server.semantic_memory.storage.text_sanitizer import sanitize_pg_text
+
+
+class TestSanitizePgText:
+    """Tests for sanitize_pg_text."""
+
+    def test_clean_string_passes_through(self):
+        assert sanitize_pg_text("hello world") == "hello world"
+
+    def test_empty_string_passes_through(self):
+        assert sanitize_pg_text("") == ""
+
+    def test_unicode_string_passes_through(self):
+        value = "Día de Los Muertos"
+        assert sanitize_pg_text(value) == value
+
+    def test_strips_single_null_byte(self):
+        assert sanitize_pg_text("D\x00a de Los Muertos") == "Da de Los Muertos"
+
+    def test_strips_multiple_null_bytes(self):
+        assert sanitize_pg_text("a\x00b\x00c") == "abc"
+
+    def test_strips_leading_null_byte(self):
+        assert sanitize_pg_text("\x00hello") == "hello"
+
+    def test_strips_trailing_null_byte(self):
+        assert sanitize_pg_text("hello\x00") == "hello"
+
+    def test_string_of_only_null_bytes(self):
+        assert sanitize_pg_text("\x00\x00\x00") == ""
+
+    def test_logs_warning_when_stripping(self, caplog):
+        with caplog.at_level(logging.WARNING):
+            sanitize_pg_text("D\x00a de Los Muertos", context="feature.value")
+
+        assert len(caplog.records) == 1
+        assert "null byte" in caplog.records[0].message.lower()
+        assert "feature.value" in caplog.records[0].message
+
+    def test_no_log_for_clean_string(self, caplog):
+        with caplog.at_level(logging.WARNING):
+            sanitize_pg_text("clean string")
+
+        assert len(caplog.records) == 0

{memmachine_server-0.3.2 → memmachine_server-0.3.3}/server_tests/memmachine_server/semantic_memory/test_semantic_ingestion.py

@@ -7,6 +7,7 @@ import numpy as np
 import pytest
 import pytest_asyncio
 
+from memmachine_server.common.data_types import ExternalServiceAPIError
 from memmachine_server.common.episode_store import (
     EpisodeEntry,
     EpisodeIdT,
@@ -212,6 +213,49 @@ async def test_process_single_set_applies_commands(
     assert embedder_double.ingest_calls == [["blue"]]
 
 
+@pytest.mark.asyncio
+async def test_process_single_set_marks_context_length_errors_as_ingested(
+    ingestion_service: IngestionService,
+    semantic_storage: SemanticStorage,
+    episode_storage: EpisodeStorage,
+    monkeypatch,
+):
+    class ContextLengthExceededError(Exception):
+        code = "context_length_exceeded"
+
+    message_id = await add_history(
+        episode_storage,
+        content="very long message that exceeds model context",
+    )
+    await semantic_storage.add_history_to_set(
+        set_id="user-context-overflow",
+        history_id=message_id,
+    )
+
+    async def mock_context_length_error(*args, **kwargs):
+        err = ContextLengthExceededError("input exceeds context window")
+        raise ExternalServiceAPIError("LLM request failed") from err
+
+    monkeypatch.setattr(
+        "memmachine_server.semantic_memory.semantic_ingestion.llm_feature_update",
+        mock_context_length_error,
+    )
+
+    await ingestion_service._process_single_set("user-context-overflow")
+
+    assert (
+        await semantic_storage.get_history_messages(
+            set_ids=["user-context-overflow"],
+            is_ingested=False,
+        )
+        == []
+    )
+    assert await semantic_storage.get_history_messages(
+        set_ids=["user-context-overflow"],
+        is_ingested=True,
+    ) == [message_id]
+
+
 @pytest.mark.asyncio
 async def test_consolidation_groups_by_tag(
     ingestion_service: IngestionService,
@@ -543,6 +587,59 @@ async def test_process_single_set_raises_in_debug_mode_for_invalid_ids(
         await ingestion_service._process_single_set("user-888")
 
 
+@pytest.mark.asyncio
+async def test_process_single_set_limits_features_sent_to_llm(
+    semantic_storage: SemanticStorage,
+    episode_storage: EpisodeStorage,
+    resource_retriever: MockResourceRetriever,
+    semantic_category: SemanticCategory,
+    monkeypatch,
+):
+    """The feature lookup for each message must pass a page_size cap so that a
+    very large profile does not cause the LLM response to overflow its token budget."""
+    message_id = await add_history(episode_storage, content="I love sushi")
+    await semantic_storage.add_history_to_set(set_id="user-222", history_id=message_id)
+
+    # Seed more features than the expected cap
+    for i in range(60):
+        await semantic_storage.add_feature(
+            set_id="user-222",
+            category_name=semantic_category.name,
+            feature=f"feature_{i}",
+            value=f"value_{i}",
+            tag="misc",
+            embedding=np.array([float(i), float(-i)]),
+        )
+
+    llm_mock = AsyncMock(return_value=[])
+    monkeypatch.setattr(
+        "memmachine_server.semantic_memory.semantic_ingestion.llm_feature_update",
+        llm_mock,
+    )
+
+    ingestion_service = IngestionService(
+        IngestionService.Params(
+            semantic_storage=semantic_storage,
+            history_store=episode_storage,
+            resource_retriever=resource_retriever.get_resources,
+            consolidated_threshold=100,
+            debug_fail_loudly=False,
+        )
+    )
+
+    await ingestion_service._process_single_set("user-222")
+
+    # get_feature_set must have been called with a non-None page_size
+    # to prevent sending all 60 features to the LLM at once.
+    llm_mock.assert_awaited_once()
+    assert llm_mock.await_args is not None
+    passed_features: list[SemanticFeature] = llm_mock.await_args.kwargs["features"]
+    assert len(passed_features) < 60, (
+        "Expected feature set to be capped before calling the LLM, "
+        f"but {len(passed_features)} features were passed"
+    )
+
+
 @pytest.mark.asyncio
 async def test_consolidation_deletes_features_not_in_keep_list(
     ingestion_service: IngestionService,