contextpress 0.5.6__tar.gz → 0.5.8__tar.gz

{contextpress-0.5.6 → contextpress-0.5.8}/AUDIT.md

@@ -26,7 +26,7 @@ Profiles: `chat`, `rag_doc`, `agent`. Tier 1 deps: nltk, scikit-learn, sumy, tik
 |----|----------|---------|--------|
 | C1 | Medium | Invariant 1 says system turns are untouched; `BudgetStrategy` truncates system as last resort (`budget.py`) | Document in contract (Phase 2) or change behavior later |
 | C2 | Medium | LLM dedup failures in `Pipeline._run_llm_tier` fall back silently (no warning); summarize path warns | Add warning (Phase 2) |
-| C3 | Medium | Filler drops ack phrases like `"sounds good"` before resolution can use them (`filler.py` / `resolution.py`) | Document + fixture test (Phases 3–4) |
+| C3 | Low | Filler drops assistant-only ack phrases like `"sounds good"` before resolution (`filler.py` / `resolution.py`) | **By design** — fixture `08_*` + Layer-A still collapses; documented |
 | C4 | Low | `Turn.importance` / `Turn.resolved` set by resolution but unused by pipeline | Leave; optional cleanup later |
 
 ### Duplication / maintainability (Phase 2)
@@ -71,8 +71,17 @@ Profiles: `chat`, `rag_doc`, `agent`. Tier 1 deps: nltk, scikit-learn, sumy, tik
 |-------|---------|--------|
 | 1 Analysis | **0.5.5** | This document + backlog — shipped |
 | 2 Refactor | **0.5.6** | R1–R3, C1 doc, C2 warning (no semantic stage changes) — shipped |
-| 3 Fixtures | **0.5.7** | Offline chat samples + smoke/invariant tests |
-| 4 Fixes | **0.5.8** | Bugs proven by fixtures + T1–T5 regression tests |
+| 3 Fixtures | **0.5.7** | Offline chat samples + smoke/invariant tests — shipped |
+| 4 Fixes | **0.5.8** | Bugs proven by fixtures + T1–T5 regression tests — shipped |
+
+### Phase 4 outcomes (0.5.8)
+
+| Fix | Detail |
+|-----|--------|
+| Filler punctuation | Cleanup orphan commas / leading punctuation after phrase removal |
+| User filler-only | User turns are not dropped when filler removal empties them |
+| `messages=None` | Raises ``TypeError`` (was silently empty) |
+| Regression tests | Extra keys, rag_doc skips resolution, resolution fixture, ``compress_many`` immutability |
 
 ## Invariants checklist (from `pipeline.py`)
 

{contextpress-0.5.6 → contextpress-0.5.8}/CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to `contextpress` are recorded here.
 The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.8] - 2026-07-26
+
+- **Stabilization Phase 4** — fixture-driven fixes:
+  - Filler stage cleans orphan commas / leading punctuation after removals.
+  - User turns are never dropped when filler removal would empty them.
+  - ``compress(None)`` / ``normalize_messages(None)`` raise ``TypeError``.
+  - Regression tests for rag_doc (no resolution), resolution fixture, extra dict keys, batch immutability.
+
+## [0.5.7] - 2026-07-26
+
+- **Stabilization Phase 3** — offline chat fixtures under `tests/fixtures/chats/` (8 synthetic threads).
+- Fixture smoke tests: compress, input immutability, preview, ``recommend_preset``.
+- Documents AUDIT C3 (filler ack vs resolution) with a dedicated fixture.
+
 ## [0.5.6] - 2026-07-26
 
 - **Stabilization Phase 2** — low-risk refactors (see [`AUDIT.md`](AUDIT.md)):

{contextpress-0.5.6 → contextpress-0.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: contextpress
-Version: 0.5.6
+Version: 0.5.8
 Summary: Deterministic context compression for LLM chat, RAG, and agent pipelines
 Project-URL: Homepage, https://github.com/Taha-azizi/contextpress
 Project-URL: Documentation, https://github.com/Taha-azizi/contextpress#readme

{contextpress-0.5.6 → contextpress-0.5.8}/contextpress/__init__.py

@@ -16,7 +16,7 @@ __all__ = [
     "CompressionResult",
     "CompressionStats",
 ]
-__version__ = "0.5.6"
+__version__ = "0.5.8"
 
 
 def __getattr__(name: str) -> Any:

{contextpress-0.5.6 → contextpress-0.5.8}/contextpress/normalizer.py

@@ -133,7 +133,7 @@ def normalize_messages(
     ctx: dict[str, Any] = {"format": "unknown", "extras": []}
 
     if messages is None:
-        return Conversation(turns=[], type=context_type, metadata={}), ctx
+        raise TypeError("contextpress: messages must not be None")
 
     # LangChain-style objects (list of objects with .content)
     if (

{contextpress-0.5.6 → contextpress-0.5.8}/contextpress/strategies/filler.py

@@ -98,13 +98,28 @@ _FILLER_RE = _build_filler_pattern()
 _ACTUALLY_NO = re.compile(r"^\s*actually\s*,\s*no\b", re.IGNORECASE)
 
 
+def _cleanup_after_filler(text: str) -> str:
+    """Repair punctuation left behind after filler phrase removal."""
+    s = re.sub(r"\s+", " ", text)
+    s = re.sub(r"\s*,\s*(?:,\s*)+", ", ", s)
+    s = re.sub(r"\s*;\s*(?:;\s*)+", "; ", s)
+    s = re.sub(r"\s+([,.;:!?])", r"\1", s)
+    s = re.sub(r"^[\s,;:.]+", "", s)
+    s = re.sub(r"[,;]\s*$", "", s)
+    s = s.strip()
+    if s and s[0].islower():
+        s = s[0].upper() + s[1:]
+    return s
+
+
 def _remove_fillers_text(text: str) -> str:
     if _ACTUALLY_NO.match(text):
         return text
     # do not strip "actually" in "actually, no"
     s = _FILLER_RE.sub("", text)
-    s = re.sub(r"\s+", " ", s).strip()
-    return s
+    if s == text:
+        return text
+    return _cleanup_after_filler(s)
 
 
 def _is_acknowledgement_only(text: str) -> bool:
@@ -152,6 +167,9 @@ class FillerStrategy(BaseStrategy):
 
             new_text = _remove_fillers_text(text)
             if not new_text.strip():
+                # Never drop user turns entirely (filler-only user messages stay).
+                if turn.role == "user":
+                    new_turns.append(copy.deepcopy(turn))
                 continue
 
             if new_text != text:

{contextpress-0.5.6 → contextpress-0.5.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "contextpress"
-version = "0.5.6"
+version = "0.5.8"
 description = "Deterministic context compression for LLM chat, RAG, and agent pipelines"
 readme = "README.md"
 license = { file = "LICENSE" }

contextpress-0.5.8/tests/fixtures/chats/01_filler_heavy.json

@@ -0,0 +1,14 @@
+{
+  "id": "01_filler_heavy",
+  "type": "chat",
+  "source": "synthetic (ShareGPT-style filler patterns)",
+  "messages": [
+    {"role": "system", "content": "You are a helpful coding assistant."},
+    {"role": "user", "content": "Basically, I was wondering if I should use Postgres or Mongo for my app?"},
+    {"role": "assistant", "content": "Honestly, it really depends. Basically Postgres is great for relational data. Just saying."},
+    {"role": "user", "content": "Sounds good. What about indexes?"},
+    {"role": "assistant", "content": "Yeah, you know, you can add B-tree indexes. Actually, for text search, consider GIN."},
+    {"role": "user", "content": "Ok cool. And migrations?"},
+    {"role": "assistant", "content": "Basically use Alembic or Flyway. Honestly that should work fine."}
+  ]
+}

contextpress-0.5.8/tests/fixtures/chats/02_resolution_thread.json

@@ -0,0 +1,16 @@
+{
+  "id": "02_resolution_thread",
+  "type": "chat",
+  "source": "synthetic (decision / agreement thread)",
+  "messages": [
+    {"role": "system", "content": "You help teams pick technologies."},
+    {"role": "user", "content": "Should we use Postgres or Mongo for the billing service?"},
+    {"role": "assistant", "content": "Postgres is a better fit for transactional billing and joins."},
+    {"role": "user", "content": "What about schema flexibility?"},
+    {"role": "assistant", "content": "JSON columns cover most flexibility needs without giving up ACID."},
+    {"role": "user", "content": "Ok let's go with Postgres then."},
+    {"role": "assistant", "content": "Agreed. Postgres it is for billing."},
+    {"role": "user", "content": "Next: how do we deploy?"},
+    {"role": "assistant", "content": "Use blue-green deploys on Kubernetes with a short drain window."}
+  ]
+}

contextpress-0.5.8/tests/fixtures/chats/03_repetition.json

@@ -0,0 +1,13 @@
+{
+  "id": "03_repetition",
+  "type": "chat",
+  "source": "synthetic (near-duplicate user/assistant turns)",
+  "messages": [
+    {"role": "user", "content": "Please explain how Python list comprehensions work with filtering and nesting examples included for beginners."},
+    {"role": "assistant", "content": "List comprehensions build lists from iterables: [x for x in items if cond]. Nested ones mirror nested loops and keep filtering close to each level."},
+    {"role": "user", "content": "Please explain how Python list comprehensions work with filtering and nesting examples included for beginners please."},
+    {"role": "assistant", "content": "List comprehensions build lists from iterables: [x for x in items if cond]. Nested ones mirror nested loops and keep filtering close to each level of iteration."},
+    {"role": "user", "content": "Can you show a dict comprehension too?"},
+    {"role": "assistant", "content": "Sure: {k: v for k, v in pairs if v is not None}."}
+  ]
+}

contextpress-0.5.8/tests/fixtures/chats/04_long_history.json

@@ -0,0 +1,20 @@
+{
+  "id": "04_long_history",
+  "type": "chat",
+  "source": "synthetic (long multi-turn, LMSYS-style length)",
+  "messages": [
+    {"role": "system", "content": "You are a concise technical assistant for web infrastructure questions."},
+    {"role": "user", "content": "Topic A: please explain HTTP in detail including methods, status codes, and common headers used in APIs today."},
+    {"role": "assistant", "content": "HTTP is a request-response protocol. Common methods are GET, POST, PUT, PATCH, DELETE. Status codes include 2xx success, 4xx client errors, 5xx server errors. Headers like Content-Type, Authorization, and Cache-Control are ubiquitous."},
+    {"role": "user", "content": "Topic B: what is TLS and how does the handshake roughly work for HTTPS websites and APIs?"},
+    {"role": "assistant", "content": "TLS encrypts traffic between clients and servers. The handshake negotiates ciphers, authenticates the server certificate, and derives session keys so application data can be encrypted."},
+    {"role": "user", "content": "Topic C: explain DNS records people usually need for apps, including A, AAAA, CNAME, and TXT."},
+    {"role": "assistant", "content": "DNS maps names to addresses and services. A/AAAA point to IPv4/IPv6, CNAME aliases names, and TXT often stores verification or SPF/DKIM related data."},
+    {"role": "user", "content": "Topic D: what problems does a CDN solve and when should I not put everything behind a CDN?"},
+    {"role": "assistant", "content": "A CDN caches static and some dynamic content near users, reducing latency and origin load. Avoid caching personalized or rapidly changing private responses without careful cache keys."},
+    {"role": "user", "content": "Topic E: summarize REST constraints and how they differ from RPC style APIs in practice."},
+    {"role": "assistant", "content": "REST emphasizes resources, uniform interfaces, and hypermedia ideas. RPC centers on procedure calls. In practice many 'REST' APIs are resource-ish HTTP JSON endpoints without full HATEOAS."},
+    {"role": "user", "content": "Back to my real question: how do I cache API responses safely for a multi-tenant SaaS with per-user data?"},
+    {"role": "assistant", "content": "Use Cache-Control with short TTLs, vary on Authorization or tenant id, prefer ETags for validation, and invalidate on writes. Never share personalized responses across tenants."}
+  ]
+}

contextpress-0.5.8/tests/fixtures/chats/05_agent_tools.json

@@ -0,0 +1,16 @@
+{
+  "id": "05_agent_tools",
+  "type": "agent",
+  "source": "synthetic (tool-using agent thread)",
+  "messages": [
+    {"role": "system", "content": "You are a deploy agent with tools."},
+    {"role": "user", "content": "Is api-v2 on staging?"},
+    {"role": "assistant", "content": "Checking status <tool_call> get_deploy_status(api-v2, staging)"},
+    {"role": "user", "content": "Tool result: version 2.4.1 healthy"},
+    {"role": "assistant", "content": "api-v2 staging is healthy on 2.4.1."},
+    {"role": "user", "content": "We've decided on using the new pipeline for deploy."},
+    {"role": "assistant", "content": "Confirmed. Scheduling via the new pipeline."},
+    {"role": "user", "content": "Any blockers?"},
+    {"role": "assistant", "content": "Basically none. Honestly looking good."}
+  ]
+}

contextpress-0.5.8/tests/fixtures/chats/06_rag_chunks.json

@@ -0,0 +1,13 @@
+{
+  "id": "06_rag_chunks",
+  "type": "rag_doc",
+  "source": "synthetic (RAG document chunks + query)",
+  "messages": [
+    {"role": "system", "content": "Answer using the provided document chunks."},
+    {"role": "user", "content": "Chunk 1: Contextpress is a Python library for deterministic LLM context compression using classical NLP stages such as filler removal, repetition detection, and token budgets."},
+    {"role": "user", "content": "Chunk 2: The package supports chat, rag_doc, and agent profiles. Resolution collapsing is disabled for rag_doc because documents are not speaker dialogues."},
+    {"role": "user", "content": "Chunk 3: Optional Tier 2 LLM backends can summarize after Tier 1. Failures fall back to Tier 1 with a warning."},
+    {"role": "user", "content": "Chunk 4: Unrelated weather note: rain is expected tomorrow in Seattle with strong winds near the coast."},
+    {"role": "user", "content": "Which profiles does contextpress support and is resolution used for rag_doc?"}
+  ]
+}

contextpress-0.5.8/tests/fixtures/chats/07_short_stable.json

@@ -0,0 +1,9 @@
+{
+  "id": "07_short_stable",
+  "type": "chat",
+  "source": "synthetic (short chat; expect minimal change)",
+  "messages": [
+    {"role": "user", "content": "What is 2+2?"},
+    {"role": "assistant", "content": "4"}
+  ]
+}

contextpress-0.5.8/tests/fixtures/chats/08_mixed_ack_resolution.json

@@ -0,0 +1,13 @@
+{
+  "id": "08_mixed_ack_resolution",
+  "type": "chat",
+  "source": "synthetic (filler ack may remove resolution signal — AUDIT C3)",
+  "messages": [
+    {"role": "user", "content": "Should we ship Friday or Monday?"},
+    {"role": "assistant", "content": "Monday is safer for the release window."},
+    {"role": "user", "content": "Sounds good"},
+    {"role": "assistant", "content": "Sounds good"},
+    {"role": "user", "content": "We'll use Monday then."},
+    {"role": "assistant", "content": "Confirmed. Monday release it is."}
+  ]
+}

contextpress-0.5.8/tests/fixtures/chats/README.md

@@ -0,0 +1,15 @@
+# Offline chat fixtures (Phase 3)
+
+Synthetic conversations inspired by public chat / RAG / agent patterns.
+No network required — used by `tests/test_fixture_chats.py`.
+
+| File | Profile | Focus |
+|------|---------|--------|
+| `01_filler_heavy.json` | chat | Filler phrases |
+| `02_resolution_thread.json` | chat | Agreement / RESOLVED |
+| `03_repetition.json` | chat | Near-duplicate turns |
+| `04_long_history.json` | chat | Long multi-topic history |
+| `05_agent_tools.json` | agent | Tool markers |
+| `06_rag_chunks.json` | rag_doc | Chunk + query |
+| `07_short_stable.json` | chat | Minimal change |
+| `08_mixed_ack_resolution.json` | chat | Ack vs resolution (AUDIT C3) |

{contextpress-0.5.6 → contextpress-0.5.8}/tests/test_filler.py

@@ -21,6 +21,23 @@ def test_filler_sentence_start():
     t = Turn(role="assistant", content="Basically, we use Postgres.")
     out = _run()([t]).turns[0]
     assert "basically" not in out.content.lower()
+    assert not out.content.startswith(",")
+    assert out.content.startswith("We")
+
+
+def test_filler_cleans_orphan_commas():
+    t = Turn(role="assistant", content="Yeah, you know, you can add indexes.")
+    out = _run()([t]).turns[0]
+    assert "you know" not in out.content.lower()
+    assert ", ," not in out.content
+    assert "Yeah, you can" in out.content
+
+
+def test_filler_only_user_not_dropped():
+    t = Turn(role="user", content="Basically.")
+    out = _run()([t]).turns
+    assert len(out) == 1
+    assert out[0].role == "user"
 
 
 def test_acknowledgement_assistant_dropped():

contextpress-0.5.8/tests/test_fixture_chats.py

@@ -0,0 +1,72 @@
+"""Smoke + invariant tests over offline chat fixtures (Phase 3)."""
+
+from __future__ import annotations
+
+import copy
+import json
+from pathlib import Path
+
+import pytest
+
+from contextpress import ContextManager
+
+FIXTURES = sorted(Path(__file__).parent.joinpath("fixtures", "chats").glob("*.json"))
+
+
+def _load(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+@pytest.mark.parametrize("path", FIXTURES, ids=[p.stem for p in FIXTURES])
+def test_fixture_compress_smoke(path: Path):
+    data = _load(path)
+    messages = data["messages"]
+    cm = ContextManager(type=data.get("type", "chat"), compression="medium")
+    result = cm.compress(messages, token_budget=800, return_stats=True)
+    assert isinstance(result.messages, list)
+    assert result.stats.tokens_after <= result.stats.tokens_before
+    assert result.stats.turns_after >= 1 or len(messages) == 0
+    assert result.stats.context_type == data.get("type", "chat")
+
+
+@pytest.mark.parametrize("path", FIXTURES, ids=[p.stem for p in FIXTURES])
+def test_fixture_input_immutable(path: Path):
+    data = _load(path)
+    original = copy.deepcopy(data["messages"])
+    messages = copy.deepcopy(data["messages"])
+    cm = ContextManager(type=data.get("type", "chat"))
+    cm.compress(messages, token_budget=400)
+    assert messages == original
+
+
+@pytest.mark.parametrize("path", FIXTURES, ids=[p.stem for p in FIXTURES])
+def test_fixture_preview_unchanged(path: Path):
+    data = _load(path)
+    messages = data["messages"]
+    cm = ContextManager(type=data.get("type", "chat"))
+    preview = cm.preview(messages, token_budget=400)
+    assert preview.messages == messages
+    assert preview.stats.dry_run is True
+
+
+@pytest.mark.parametrize("path", FIXTURES, ids=[p.stem for p in FIXTURES])
+def test_fixture_recommend_preset(path: Path):
+    data = _load(path)
+    cm = ContextManager(type=data.get("type", "chat"))
+    preset = cm.recommend_preset(data["messages"], token_budget=500)
+    assert preset in ("low", "medium", "high")
+
+
+def test_fixture_catalog_not_empty():
+    assert len(FIXTURES) >= 8
+
+
+def test_c3_ack_may_weaken_resolution_signal():
+    """Document AUDIT C3: filler can drop ack-only turns before resolution."""
+    data = _load(Path(__file__).parent / "fixtures" / "chats" / "08_mixed_ack_resolution.json")
+    cm = ContextManager(type="chat", compression="high")
+    result = cm.compress(data["messages"], token_budget=None, return_stats=True)
+    # Still a valid pipeline outcome; may or may not emit RESOLVED depending on remaining signals.
+    texts = " ".join(str(m.get("content", "")) for m in result.messages)
+    assert result.stats.turns_after <= result.stats.turns_before
+    assert "Monday" in texts or "RESOLVED" in texts

contextpress-0.5.8/tests/test_v058.py

@@ -0,0 +1,55 @@
+import copy
+import json
+from pathlib import Path
+
+import pytest
+
+from contextpress import ContextManager
+
+
+def test_messages_none_raises():
+    cm = ContextManager(type="chat")
+    with pytest.raises(TypeError, match="must not be None"):
+        cm.compress(None)
+
+
+def test_extra_dict_keys_preserved():
+    cm = ContextManager(type="chat")
+    messages = [{"role": "user", "content": "hello basically", "name": "alice", "custom": 123}]
+    out = cm.compress(messages, token_budget=None)
+    assert out[0]["name"] == "alice"
+    assert out[0]["custom"] == 123
+
+
+def test_rag_doc_skips_resolution_stage():
+    data = json.loads(
+        (Path(__file__).parent / "fixtures" / "chats" / "06_rag_chunks.json").read_text(
+            encoding="utf-8"
+        )
+    )
+    cm = ContextManager(type="rag_doc", compression="high")
+    result = cm.compress(data["messages"], token_budget=None, return_stats=True)
+    assert "resolution" not in result.stats.stages_run
+    assert not any("RESOLVED" in str(m.get("content", "")) for m in result.messages)
+
+
+def test_resolution_fixture_collapses():
+    data = json.loads(
+        (Path(__file__).parent / "fixtures" / "chats" / "02_resolution_thread.json").read_text(
+            encoding="utf-8"
+        )
+    )
+    cm = ContextManager(type="chat", compression="high")
+    result = cm.compress(data["messages"], token_budget=None, return_stats=True)
+    assert any("RESOLVED" in str(m.get("content", "")) for m in result.messages)
+
+
+def test_compress_many_immutability():
+    batches = [
+        [{"role": "user", "content": "hello basically"}],
+        [{"role": "user", "content": "thanks basically"}],
+    ]
+    original = copy.deepcopy(batches)
+    cm = ContextManager(type="chat")
+    cm.compress_many(batches, token_budget=200)
+    assert batches == original