virtual-context 0.2.6__tar.gz → 0.2.8__tar.gz

{virtual_context-0.2.6 → virtual_context-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: virtual-context
-Version: 0.2.6
+Version: 0.2.8
 Summary: OS-style virtual memory for LLM session context management
 Project-URL: Homepage, https://virtual-context.com
 Project-URL: Repository, https://github.com/virtual-context/virtual-context
@@ -46,8 +46,11 @@ Requires-Dist: redis>=5.0; extra == 'redis'
 Description-Content-Type: text/markdown
 
 <p align="center">
-  <img src="assets/hero.png" alt="virtual-context" width="100%">
+  <a href="assets/dashboard.png">
+    <img src="assets/dashboard.png" alt="virtual-context dashboard" width="800">
+  </a>
 </p>
+<p align="center"><sub>virtual-context cloud - running 3 million virtual token window at 80k actual tokens</sub></p>
 
 # virtual-context
 

{virtual_context-0.2.6 → virtual_context-0.2.8}/README.md

@@ -1,6 +1,9 @@
 <p align="center">
-  <img src="assets/hero.png" alt="virtual-context" width="100%">
+  <a href="assets/dashboard.png">
+    <img src="assets/dashboard.png" alt="virtual-context dashboard" width="800">
+  </a>
 </p>
+<p align="center"><sub>virtual-context cloud - running 3 million virtual token window at 80k actual tokens</sub></p>
 
 # virtual-context
 

{virtual_context-0.2.6 → virtual_context-0.2.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "virtual-context"
-version = "0.2.6"
+version = "0.2.8"
 description = "OS-style virtual memory for LLM session context management"
 readme = "README.md"
 license = "AGPL-3.0-or-later"

virtual_context-0.2.8/tests/test_fill_pass.py

@@ -0,0 +1,274 @@
+"""Tests for the fill pass pipeline stage."""
+
+import copy
+import json
+import re
+from unittest.mock import MagicMock
+from datetime import datetime, timezone
+
+from virtual_context.proxy.formats import detect_format
+from virtual_context.types import (
+    RetrievalResult, StoredSummary, AssembledContext, TagSummary, SegmentMetadata,
+)
+
+
+def test_retrieval_result_has_overflow_field():
+    rr = RetrievalResult()
+    assert hasattr(rr, "overflow_summaries")
+    assert rr.overflow_summaries == []
+
+
+def test_retrieval_result_overflow_accepts_summaries():
+    s = StoredSummary(ref="seg_001", primary_tag="cooking", summary="test", summary_tokens=50)
+    rr = RetrievalResult(overflow_summaries=[s])
+    assert len(rr.overflow_summaries) == 1
+    assert rr.overflow_summaries[0].ref == "seg_001"
+
+
+def test_assembled_context_has_retrieval_result():
+    ac = AssembledContext()
+    assert ac.retrieval_result is None
+
+
+def test_assembled_context_has_presented_tags():
+    ac = AssembledContext()
+    assert isinstance(ac.presented_tags, set)
+    assert len(ac.presented_tags) == 0
+
+
+def test_assembled_context_presented_tags_populated():
+    ac = AssembledContext(presented_tags={"cooking", "baking", "recipes"})
+    assert "cooking" in ac.presented_tags
+    assert len(ac.presented_tags) == 3
+
+
+def test_format_tag_section_standalone():
+    """Standalone format_tag_section produces the same XML format as the assembler."""
+    from virtual_context.core.assembler import format_tag_section
+    from virtual_context.types import StoredSummary, SegmentMetadata
+    from datetime import datetime, timezone
+
+    s1 = StoredSummary(
+        ref="s1", primary_tag="cooking", summary="Italian cooking techniques",
+        summary_tokens=100, tags=["cooking", "italian"],
+        metadata=SegmentMetadata(),
+        start_timestamp=datetime.now(timezone.utc),
+    )
+
+    result = format_tag_section("cooking", [s1])
+    assert '<virtual-context tags="cooking, italian"' in result
+    assert "[1/1]" in result
+    assert "Italian cooking techniques" in result
+    assert "</virtual-context>" in result
+
+
+def _make_summary(ref: str, tag: str, tokens: int) -> StoredSummary:
+    return StoredSummary(
+        ref=ref, primary_tag=tag, tags=[tag],
+        summary=f"Summary for {tag}", summary_tokens=tokens,
+        metadata=SegmentMetadata(),
+        start_timestamp=datetime.now(timezone.utc),
+    )
+
+
+def _make_anthropic_body(user_texts: list[str]) -> dict:
+    messages = []
+    for text in user_texts:
+        messages.append({"role": "user", "content": text})
+        messages.append({"role": "assistant", "content": [{"type": "text", "text": f"Reply to: {text}"}]})
+    messages.append({"role": "user", "content": "current question"})
+    return {"system": "You are helpful.", "messages": messages, "model": "claude-opus-4-6"}
+
+
+def test_fill_pass_no_op_when_at_target():
+    from virtual_context.proxy.message_filter import fill_pass
+    body = _make_anthropic_body(["hello", "world"])
+    fmt = detect_format(body)
+    result, summaries, turns = fill_pass(
+        body=body, fmt=fmt, outbound_tokens=90000, target_tokens=90000,
+        assembled=None, pre_filter_body=body,
+        store=MagicMock(), conversation_id="test",
+    )
+    assert summaries == 0
+    assert turns == 0
+
+
+def test_fill_pass_no_op_when_over_target():
+    from virtual_context.proxy.message_filter import fill_pass
+    body = _make_anthropic_body(["hello"])
+    fmt = detect_format(body)
+    result, summaries, turns = fill_pass(
+        body=body, fmt=fmt, outbound_tokens=100000, target_tokens=90000,
+        assembled=None, pre_filter_body=body,
+        store=MagicMock(), conversation_id="test",
+    )
+    assert summaries == 0
+    assert turns == 0
+
+
+def test_fill_pass_adds_breadth_summaries():
+    from virtual_context.proxy.message_filter import fill_pass
+
+    body = _make_anthropic_body(["hello"])
+    fmt = detect_format(body)
+
+    ts1 = TagSummary(tag="cooking", summary="Italian cooking", summary_tokens=50,
+                     source_segment_refs=["seg_a"], updated_at=datetime.now(timezone.utc))
+    ts2 = TagSummary(tag="baking", summary="Bread baking", summary_tokens=50,
+                     source_segment_refs=["seg_b"], updated_at=datetime.now(timezone.utc))
+    mock_store = MagicMock()
+    mock_store.get_all_tag_summaries.return_value = [ts1, ts2]
+
+    assembled = AssembledContext(
+        presented_segment_refs=set(),
+        presented_tags=set(),
+        tag_sections={},
+        retrieval_result=RetrievalResult(),
+    )
+
+    result, summaries_added, turns_added = fill_pass(
+        body=body, fmt=fmt, outbound_tokens=70000, target_tokens=90000,
+        assembled=assembled, pre_filter_body=copy.deepcopy(body),
+        store=mock_store, conversation_id="test",
+        summary_ratio=1.0,
+    )
+    assert summaries_added >= 1
+    mock_store.get_all_tag_summaries.assert_called_once_with(conversation_id="test")
+
+
+def test_presented_tags_from_segments_and_full_sections():
+    """presented_tags must include secondary tags from SEGMENTS/FULL sections."""
+    tag_sections = {
+        "cooking": (
+            '<virtual-context tags="cooking, italian" segments="2">\n'
+            "[1/2]\nPasta techniques\n\n---\n\n[2/2]\nRisotto methods\n"
+            "</virtual-context>"
+        ),
+        "baking": (
+            '<virtual-context tags="baking, bread, sourdough" segments="1">\n'
+            "[1/1]\nSourdough starter maintenance\n"
+            "</virtual-context>"
+        ),
+    }
+
+    _vc_tags_re = re.compile(r'<virtual-context\s+tags="([^"]*)"')
+    presented_tags: set[str] = set()
+    for section_text in tag_sections.values():
+        for m in _vc_tags_re.finditer(section_text):
+            for t in m.group(1).split(", "):
+                t = t.strip()
+                if t:
+                    presented_tags.add(t)
+    # Also include section keys
+    presented_tags.update(tag_sections.keys())
+
+    assert "cooking" in presented_tags
+    assert "baking" in presented_tags
+    assert "italian" in presented_tags
+    assert "bread" in presented_tags
+    assert "sourdough" in presented_tags
+    assert "grilling" not in presented_tags
+
+
+def test_presented_tags_fallback_to_section_key():
+    """When tags attr is empty, section key should still be covered."""
+    tag_sections = {
+        "misc": '<virtual-context tags="" segments="1">\n[1/1]\nRandom\n</virtual-context>',
+    }
+
+    _vc_tags_re = re.compile(r'<virtual-context\s+tags="([^"]*)"')
+    presented_tags: set[str] = set()
+    for section_text in tag_sections.values():
+        for m in _vc_tags_re.finditer(section_text):
+            for t in m.group(1).split(", "):
+                t = t.strip()
+                if t:
+                    presented_tags.add(t)
+    presented_tags.update(tag_sections.keys())
+
+    # "misc" comes from keys(), not from regex (empty attr)
+    assert "misc" in presented_tags
+
+
+def test_fill_pass_accounting_summary_and_turns():
+    """After fill pass, counts must match actual content added."""
+    from virtual_context.proxy.message_filter import fill_pass
+
+    body = _make_anthropic_body(["hello", "world"])
+    pre_filter = copy.deepcopy(body)
+    extra_msgs = [
+        {"role": "user", "content": "older message 1"},
+        {"role": "assistant", "content": [{"type": "text", "text": "older reply 1"}]},
+        {"role": "user", "content": "older message 2"},
+        {"role": "assistant", "content": [{"type": "text", "text": "older reply 2"}]},
+    ]
+    pre_filter["messages"] = extra_msgs + pre_filter["messages"]
+
+    fmt = detect_format(body)
+
+    ts1 = TagSummary(tag="history", summary="Historical events discussed",
+                     summary_tokens=30, source_segment_refs=["seg_h1"],
+                     updated_at=datetime.now(timezone.utc))
+    mock_store = MagicMock()
+    mock_store.get_all_tag_summaries.return_value = [ts1]
+
+    assembled = AssembledContext(
+        presented_segment_refs=set(),
+        presented_tags=set(),
+        tag_sections={},
+        retrieval_result=RetrievalResult(),
+    )
+
+    result_body, summaries_added, turns_added = fill_pass(
+        body=body, fmt=fmt,
+        outbound_tokens=70000, target_tokens=90000,
+        assembled=assembled, pre_filter_body=pre_filter,
+        store=mock_store, conversation_id="test-conv",
+        summary_ratio=0.5,
+    )
+
+    assert summaries_added >= 1
+    result_json = json.dumps(result_body)
+    assert "Historical events discussed" in result_json
+
+
+def test_fill_pass_sanitizes_restored_turns():
+    """Restored turns must have thinking blocks stripped and media replaced."""
+    from virtual_context.proxy.message_filter import _sanitize_restored_turn
+
+    messages = [
+        {"role": "user", "content": [
+            {"type": "text", "text": "show me the image"},
+            {"type": "image", "source": {"type": "base64", "data": "abc123"}},
+        ]},
+        {"role": "assistant", "content": [
+            {"type": "thinking", "thinking": "let me think..."},
+            {"type": "text", "text": "here is my answer"},
+        ]},
+    ]
+
+    sanitized = _sanitize_restored_turn(messages)
+
+    user_content = sanitized[0]["content"]
+    assert any(b.get("text") == "[image removed from restored turn]" for b in user_content)
+    assert not any(b.get("type") == "image" for b in user_content)
+
+    asst_content = sanitized[1]["content"]
+    assert not any(b.get("type") == "thinking" for b in asst_content)
+    assert any(b.get("text") == "here is my answer" for b in asst_content)
+
+
+def test_fill_pass_no_restore_tool_references():
+    """Media placeholders must NOT reference vc_restore_tool."""
+    from virtual_context.proxy.message_filter import _sanitize_restored_turn
+
+    messages = [
+        {"role": "user", "content": [
+            {"type": "input_image", "image_url": "data:image/png;base64,abc123"},
+        ]},
+    ]
+
+    sanitized = _sanitize_restored_turn(messages)
+    result_json = json.dumps(sanitized)
+    assert "vc_restore_tool" not in result_json
+    assert "image removed from restored turn" in result_json

{virtual_context-0.2.6 → virtual_context-0.2.8}/virtual_context/__init__.py

@@ -14,7 +14,7 @@ from .types import (
     VirtualContextConfig,
 )
 
-__version__ = "0.2.6"
+__version__ = "0.2.8"
 
 __all__ = [
     "VirtualContextEngine",

{virtual_context-0.2.6 → virtual_context-0.2.8}/virtual_context/conversation_identity.py

@@ -314,12 +314,16 @@ def resolve_conversation_id(
     """
     if explicit_id and explicit_id.strip():
         explicit_id = explicit_id.strip()
-        if format_name:
-            return _candidate_to_uuid("explicit_id", f"{format_name}:{explicit_id}")
+        # If explicit_id is already a valid UUID (e.g. from a vc:conversation
+        # marker in a previous response), use it as-is.  The marker IS the
+        # canonical conversation ID — re-hashing it with format_name would
+        # break the round-trip and create orphan conversations.
         try:
             _uuid.UUID(explicit_id)
             return explicit_id
         except ValueError:
+            if format_name:
+                return _candidate_to_uuid("explicit_id", f"{format_name}:{explicit_id}")
             return _candidate_to_uuid("explicit_id", explicit_id)
 
     if not body:

{virtual_context-0.2.6 → virtual_context-0.2.8}/virtual_context/core/assembler.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import fnmatch
 import logging
+import re
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Callable
@@ -24,6 +25,60 @@ from ..types import (
 )
 
 
+def format_tag_section(
+    tag: str,
+    summaries: list["StoredSummary"],
+    store: "ContextStore | None" = None,
+    conversation_id: str = "",
+) -> str:
+    """Render a tag section in the canonical <virtual-context> format.
+
+    Shared by the assembler and the fill pass. Tool-hint enrichment is
+    optional (gracefully degrades if store is None).
+    """
+    if not summaries:
+        return ""
+
+    summaries = sorted(summaries, key=lambda s: s.start_timestamp)
+
+    all_tags = sorted({t for s in summaries for t in s.tags})
+    tags_attr = ", ".join(all_tags) if all_tags else tag
+
+    total = len(summaries)
+    summary_texts: list[str] = []
+    for idx, s in enumerate(summaries, 1):
+        prefix = f"[{idx}/{total}]"
+        session = s.metadata.session_date
+        if session:
+            prefix += f" [{session}]"
+        text = f"{prefix}\n{s.summary}"
+        tool_tags = [t for t in s.tags if t.startswith("tool_")]
+        if tool_tags:
+            text += f'\n[tool output truncated — vc_expand_topic("{tool_tags[0]}") for full result]'
+        # Optional tool hint enrichment
+        if store and conversation_id and s.ref:
+            get_refs = getattr(store, "get_tool_outputs_for_segment", None)
+            get_names = getattr(store, "get_tool_names_for_segment", None)
+            if callable(get_refs) and callable(get_names):
+                try:
+                    refs = get_refs(conversation_id, s.ref)
+                    if refs:
+                        names = get_names(conversation_id, s.ref)
+                        names_str = ", ".join(names) if names else "tools"
+                        text += f"\n[Tools: {names_str} -- {len(refs)} outputs restorable via vc_restore_tool]"
+                except Exception:
+                    pass
+        summary_texts.append(text)
+
+    body = "\n\n---\n\n".join(summary_texts)
+
+    return (
+        f'<virtual-context tags="{tags_attr}" segments="{len(summaries)}">\n'
+        f"{body}\n"
+        f"</virtual-context>"
+    )
+
+
 class ContextAssembler:
     """Assemble enriched context within token budget.
 
@@ -275,6 +330,18 @@ class ContextAssembler:
 
         total_tokens = core_tokens + tag_tokens + facts_tokens_actual + conv_tokens
 
+        # Compute presented_tags from rendered <virtual-context tags="..."> headers
+        _vc_tags_re = re.compile(r'<virtual-context\s+tags="([^"]*)"')
+        _presented_tags: set[str] = set()
+        for _section_text in tag_sections.values():
+            for _m in _vc_tags_re.finditer(_section_text):
+                for _t in _m.group(1).split(", "):
+                    _t = _t.strip()
+                    if _t:
+                        _presented_tags.add(_t)
+        # Also include section keys as baseline for edge cases
+        _presented_tags.update(tag_sections.keys())
+
         return AssembledContext(
             core_context=core,
             tag_sections=tag_sections,
@@ -291,6 +358,8 @@ class ContextAssembler:
             prepend_text=prepend_text,
             presented_segment_refs=presented_refs,
             selected_facts=selected_facts,
+            retrieval_result=retrieval_result,
+            presented_tags=_presented_tags,
         )
 
     def _format_facts(self, facts: list[Fact], max_tokens: int) -> str:
@@ -313,39 +382,7 @@ class ContextAssembler:
         return "<facts>\n" + "\n".join(lines) + "\n</facts>"
 
     def _format_tag_section(self, tag: str, summaries: list[StoredSummary]) -> str:
-        if not summaries:
-            return ""
-
-        # Sort chronologically so reader sees old → new progression
-        summaries = sorted(summaries, key=lambda s: s.start_timestamp)
-
-        all_tags = sorted({t for s in summaries for t in s.tags})
-        tags_attr = ", ".join(all_tags) if all_tags else tag
-
-        # Prefix each summary with sequence number and optional session date
-        total = len(summaries)
-        summary_texts: list[str] = []
-        for idx, s in enumerate(summaries, 1):
-            prefix = f"[{idx}/{total}]"
-            session = s.metadata.session_date
-            if session:
-                prefix += f" [{session}]"
-            text = f"{prefix}\n{s.summary}"
-            # Expansion hint for tool segments
-            tool_tags = [t for t in s.tags if t.startswith("tool_")]
-            if tool_tags:
-                text += f'\n[tool output truncated — vc_expand_topic("{tool_tags[0]}") for full result]'
-            # Tool-enriched segment summary: append tool hint at assembly time
-            text = self._maybe_append_tool_hint(text, s.ref)
-            summary_texts.append(text)
-
-        body = "\n\n---\n\n".join(summary_texts)
-
-        return (
-            f'<virtual-context tags="{tags_attr}" segments="{len(summaries)}">\n'
-            f"{body}\n"
-            f"</virtual-context>"
-        )
+        return format_tag_section(tag, summaries, store=self._store, conversation_id=self._conversation_id)
 
     def _format_segments_section(self, tag: str, segments: list[StoredSegment]) -> str:
         if not segments:

{virtual_context-0.2.6 → virtual_context-0.2.8}/virtual_context/core/retriever.py

@@ -120,6 +120,7 @@ class ContextRetriever:
         """
         start_time = time.monotonic()
         active_tags = set(current_active_tags or [])
+        overflow: list[StoredSummary] = []
 
         # Tag the inbound message — scope vocabulary to current conversation
         store_tags = [ts.tag for ts in self.store.get_all_tags(
@@ -207,6 +208,7 @@ class ContextRetriever:
                         summaries=floor_summaries,
                         total_tokens=floor_tokens,
                         facts=self._fetch_all_facts(),
+                        overflow_summaries=overflow,
                         retrieval_metadata={
                             "elapsed_ms": round(elapsed * 1000, 1),
                             "tags_from_message": tag_result.tags,
@@ -226,6 +228,7 @@ class ContextRetriever:
                 summaries=[],
                 total_tokens=0,
                 facts=self._fetch_all_facts(),
+                overflow_summaries=overflow,
                 retrieval_metadata={
                     "elapsed_ms": round(elapsed * 1000, 1),
                     "tags_from_message": tag_result.tags,
@@ -293,15 +296,18 @@ class ContextRetriever:
         selected: list[StoredSummary] = []
         selected_refs: set[str] = set()
         total_tokens = 0
+        _overflow_cap = 50
         for summary in all_summaries:
             if summary.ref in selected_refs:
                 continue
             if total_tokens + summary.summary_tokens > token_budget:
-                logger.info(
-                    "Retriever: '%s' SKIP (budget exhausted: need %dt, have %dt remaining of %dt)",
-                    summary.primary_tag, summary.summary_tokens, token_budget - total_tokens, token_budget,
-                )
-                break
+                if len(overflow) < _overflow_cap:
+                    overflow.append(summary)
+                    logger.debug(
+                        "Retriever: '%s' OVERFLOW (%dt, %d candidates)",
+                        summary.primary_tag, summary.summary_tokens, len(overflow),
+                    )
+                continue
             selected.append(summary)
             selected_refs.add(summary.ref)
             total_tokens += summary.summary_tokens
@@ -401,6 +407,7 @@ class ContextRetriever:
             full_detail=full_detail,
             total_tokens=total_tokens,
             facts=facts,
+            overflow_summaries=overflow,
             retrieval_metadata=retrieval_metadata,
             retrieval_scores=retrieval_scores,
             query_embedding=query_embedding,