patchling 0.9.0__tar.gz → 0.9.1__tar.gz

{patchling-0.9.0 → patchling-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchling
-Version: 0.9.0
+Version: 0.9.1
 Summary: Natural-language code transformation as a library: generate unified diffs with LLMs and apply them resiliently with smartapply. Formerly gptdiff.
 Author: 255labs
 License: MIT

{patchling-0.9.0 → patchling-0.9.1}/patchling/__init__.py

@@ -29,7 +29,7 @@ from .relevance import (
     filter_relevant,
 )
 
-__version__ = "0.9.0"
+__version__ = "0.9.1"
 
 __all__ = [
     "__version__",

{patchling-0.9.0 → patchling-0.9.1}/patchling/core.py

@@ -21,6 +21,7 @@ from threading import Lock
 import shutil
 import base64
 import mimetypes
+import weakref
 
 import openai
 from openai import OpenAI
@@ -377,7 +378,7 @@ def call_llm(
         if reasoning_effort:
             request["reasoning_effort"] = reasoning_effort
         try:
-            return client.chat.completions.create(**request)
+            response = client.chat.completions.create(**request)
         except TypeError as exc:
             # Retry once, without the parameter, only when this call sent it
             # and the error says the parameter is unsupported. A generic
@@ -385,8 +386,51 @@ def call_llm(
             # reasoning_effort is made exactly once.
             if "reasoning_effort" in request and _provider_rejected_reasoning_effort(exc):
                 request.pop("reasoning_effort", None)
-                return client.chat.completions.create(**request)
+                response = client.chat.completions.create(**request)
+                # The caller asked for an effort this SDK refused. Record that
+                # nothing was sent so a relevance decision is not cached under
+                # the requested effort.
+                note_effort_sent(response, "")
+                return response
             raise
+        if "reasoning_effort" in request:
+            note_effort_sent(response, request["reasoning_effort"])
+        return response
+
+
+# Responses are not always able to grow an attribute (some SDK models are
+# slotted). The weak map covers ordinary objects; the attribute covers the rest.
+_effort_sent_for_response: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary()
+
+
+def note_effort_sent(response: Any, effort: str | None) -> None:
+    """Record the ``reasoning_effort`` value actually sent for ``response``.
+
+    ``""`` means the parameter was omitted after the client rejected it.
+    """
+    value = "" if not effort else str(effort)
+    try:
+        _effort_sent_for_response[response] = value
+    except TypeError:
+        pass
+    try:
+        object.__setattr__(response, "_patchling_effort_sent", value)
+    except (AttributeError, TypeError):
+        pass
+
+
+def effort_sent_for(response: Any):
+    """Effort recorded by :func:`note_effort_sent`, or ``None`` if unrecorded."""
+    if response is None:
+        return None
+    recorded = None
+    try:
+        recorded = _effort_sent_for_response.get(response)
+    except TypeError:
+        recorded = None
+    if recorded is None:
+        recorded = getattr(response, "_patchling_effort_sent", None)
+    return recorded
 
 
 def _provider_rejected_reasoning_effort(exc: BaseException) -> bool:

{patchling-0.9.0 → patchling-0.9.1}/patchling/relevance.py

@@ -7,14 +7,18 @@ to the task. Only ``yes`` files are returned for ``generate_diff`` and
 
 Prompt version 3. Each request uses 1024 output tokens and
 ``reasoning_effort="low"`` (one retry without that parameter on a 4xx that
-names it). The cache key is the SHA-256 of length-prefixed fields: prompt
-version, model, token limit, the effort actually sent (``low``, or empty when
-the retry omitted it), task, path, mode (``full``, ``truncated``, or
-``binary``), the preview note, and the exact body sent. A decision is a
-leading standalone yes/no after the supported wrappers. Anything else is
-malformed: the file is included, ``failed_open`` is set, and the decision is
-not cached. A cache read error is a miss and the model is called; a cache
-write error is logged and does not abort the run.
+names it, or when the OpenAI client rejects the keyword). The cache key is
+the SHA-256 of length-prefixed fields: prompt version, model, token limit,
+the effort actually sent (``low``, or empty when the retry omitted it), task,
+path, mode (``full``, ``truncated``, or ``binary``), the preview note, and
+the exact body sent. A decision is a leading standalone yes/no after the
+supported wrappers, including a quoted decision followed by a reason.
+Anything else is malformed: the file is included, ``failed_open`` is set, and
+the decision is not cached. Non-string message content is that same failure.
+A cache read error is a miss and the model is called; a cache write error is
+logged and does not abort the run. Invalid UTF-8 anywhere in a byte payload
+is binary. ``on_binary="include"`` returns those original bytes unchanged.
+A leading UTF-8 BOM (U+FEFF) stays in decoded text.
 """
 
 from __future__ import annotations
@@ -625,24 +629,45 @@ def _utf8_text_or_none(sample: bytes) -> Optional[str]:
 
 
 def is_binary_content(content) -> bool:
-    """True when the first 8192 bytes are not text.
+    """True when the payload is not text.
 
-    A multibyte character cut by that window is still text. A NUL or any
-    genuinely invalid byte is binary. A non-empty buffer that decodes to
-    empty text (only an incomplete UTF-8 sequence) is binary.
+    The first 8192 bytes are decoded as UTF-8. An incomplete sequence of at
+    most 3 bytes at that probe boundary is still text. A NUL or any other
+    invalid byte inside the window is binary, and so is a non-empty window
+    that decodes to empty text. Bytes past the window are checked too: any
+    invalid UTF-8 there is binary. A leading BOM is text.
     """
     if isinstance(content, str):
-        sample = content.encode("utf-8")[:8192]
+        if "\x00" in content:
+            return True
+        raw = content.encode("utf-8")
     elif isinstance(content, (bytes, bytearray)):
-        sample = bytes(content[:8192])
+        raw = bytes(content)
     else:
         return False
-    text = _utf8_text_or_none(sample)
+    return _bytes_are_binary(raw)
+
+
+def _bytes_are_binary(raw: bytes) -> bool:
+    if b"\x00" in raw:
+        return True
+    window = raw[:8192]
+    text = _utf8_text_or_none(window)
     if text is None:
         return True
-    if sample and text == "":
+    if window and text == "":
+        return True
+    if _text_looks_binary(text):
         return True
-    return _text_looks_binary(text)
+    if len(raw) <= 8192:
+        return False
+    # The probe may end inside a character. The rest of the file has to be
+    # well-formed UTF-8; an invalid byte after the window is not dropped.
+    try:
+        raw.decode("utf-8")
+    except UnicodeDecodeError:
+        return True
+    return False
 
 
 def _text_looks_binary(sample: str) -> bool:
@@ -666,23 +691,39 @@ def _coerce_text(content) -> str:
     raise TypeError(f"file content must be str or bytes, got {type(content).__name__}")
 
 
+def _decode_utf8_text(raw: bytes) -> Optional[str]:
+    """Decode UTF-8, keeping a leading BOM (U+FEFF).
+
+    ``utf-8-sig`` is not used. A buffer of at most 8192 bytes may end on an
+    incomplete sequence; those trailing bytes are dropped. A longer buffer
+    must be entirely well-formed, so a bad byte after the probe is not cut off.
+    """
+    try:
+        return raw.decode("utf-8")
+    except UnicodeDecodeError:
+        if len(raw) > 8192:
+            return None
+        return _utf8_text_or_none(raw)
+
+
 def _prepare_content(content) -> tuple[bool, str]:
     """Return ``(is_binary, text)``.
 
-    Valid UTF-8 bytes are decoded to text and are never replaced with an
-    empty string. Invalid UTF-8 or a NUL byte takes the binary path.
+    Valid UTF-8 bytes are decoded to text, including a leading U+FEFF, and
+    are never replaced with an empty string. Invalid UTF-8 anywhere, or a
+    NUL byte, takes the binary path. Callers that include a binary file pass
+    the original bytes through instead of this text.
     """
     if isinstance(content, str):
         return is_binary_content(content), content
     if isinstance(content, (bytes, bytearray)):
         raw = bytes(content)
-        if b"\x00" in raw:
+        if is_binary_content(raw):
             return True, ""
-        try:
-            text = raw.decode("utf-8")
-        except UnicodeDecodeError:
+        text = _decode_utf8_text(raw)
+        if text is None or (raw and text == ""):
             return True, ""
-        return _text_looks_binary(text), text
+        return False, text
     raise TypeError(f"file content must be str or bytes, got {type(content).__name__}")
 
 
@@ -709,6 +750,11 @@ def _usage_triplet(usage) -> tuple[int, int, int]:
 
 
 def _content_text(content):
+    """Message content must already be a string.
+
+    A list such as ``['no']`` is not joined or passed through ``str``. That
+    would become the cacheable word ``no``. Non-string content fails open.
+    """
     if content is None:
         return ""
     if not isinstance(content, str):
@@ -878,6 +924,13 @@ def _ask(
         else:
             raise
     raw, prompt_tokens, completion_tokens, total_tokens = _usage_and_text(response)
+    # call_llm swallows a legacy SDK TypeError and retries with no effort.
+    # That response records effort "" so the decision is not stored as "low".
+    from .core import effort_sent_for
+
+    recorded = effort_sent_for(response)
+    if recorded is not None:
+        sent_effort = recorded or None
     return raw, prompt_tokens, completion_tokens, total_tokens, sent_effort
 
 
@@ -918,11 +971,15 @@ def filter_relevant(
     the empty effort. Empty answers, malformed answers, and errors are not
     cached. A cache read error is a miss and the model is called; a cache
     write error is logged and ignored. A response with missing choices, an
-    empty choice list, a missing message, or a non-JSON body fails that file
-    open and is not cached.
+    empty choice list, a missing message, non-string message content, or a
+    non-JSON body fails that file open and is not cached. When the default
+    client retries without ``reasoning_effort``, the decision is stored under
+    the empty effort.
 
     Binary files and files over ``max_bytes`` / ``max_tokens`` are not sent in
-    full. ``on_binary`` and ``on_oversize`` are ``include`` (keep, no model
+    full. Invalid UTF-8 after the first 8192 bytes is still binary.
+    ``on_binary="include"`` keeps the original bytes. ``on_binary`` and
+    ``on_oversize`` are ``include`` (keep, no model
     call), ``exclude`` (drop, no model call), or ``truncate`` (ask the model
     about a truncated or redacted preview). Pass ``None`` for ``max_bytes`` or
     ``max_tokens`` to disable that limit. ``skip_patterns`` defaults to
@@ -960,6 +1017,9 @@ def filter_relevant(
 
     by_path: dict[str, RelevanceDecision] = {}
     text_by_path: dict[str, str] = {}
+    # Binary includes keep the caller's object. Decoding would replace or drop
+    # the invalid byte and the bytes that follow it.
+    binary_original: dict = {}
     # Group files that share a cache key so each distinct body is sent once.
     groups: dict[str, list] = {}
     group_order: list[str] = []
@@ -976,6 +1036,8 @@ def filter_relevant(
         binary, text = _prepare_content(content)
         if not binary:
             text_by_path[path] = text
+        else:
+            binary_original[path] = content
         if binary:
             if on_binary == "truncate":
                 note = (
@@ -1215,7 +1277,9 @@ def filter_relevant(
     for path, content in files.items():
         decision = by_path[path]
         if decision.decision == "yes":
-            if path in text_by_path:
+            if path in binary_original:
+                included[path] = binary_original[path]
+            elif path in text_by_path:
                 included[path] = text_by_path[path]
             elif isinstance(content, str):
                 included[path] = content

{patchling-0.9.0 → patchling-0.9.1}/patchling.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchling
-Version: 0.9.0
+Version: 0.9.1
 Summary: Natural-language code transformation as a library: generate unified diffs with LLMs and apply them resiliently with smartapply. Formerly gptdiff.
 Author: 255labs
 License: MIT

{patchling-0.9.0 → patchling-0.9.1}/tests/test_relevance.py

@@ -1217,6 +1217,13 @@ def test_parser_wrappers_and_terminators():
     assert parse_relevance_answer("**yes**, because it is")[0] == "yes"
 
 
+def test_parser_quoted_decision_followed_by_reason():
+    assert parse_relevance_answer("**no** unrelated")[0] == "no"
+    assert parse_relevance_answer('"no" unrelated')[0] == "no"
+    assert parse_relevance_answer("\u201cno\u201d — unrelated")[0] == "no"
+    assert parse_relevance_answer("**yes** the helper is here")[0] == "yes"
+
+
 def test_full_and_truncated_abc_do_not_share_a_key():
     full = RelevanceCache.make_key(
         model="m", task="task", content="abc", path="a.py", mode="full", note=""
@@ -1314,6 +1321,217 @@ def test_valid_utf8_bytes_are_decoded_and_do_not_share_a_path_key():
     assert len(bodies) == 2
 
 
+def test_invalid_utf8_after_probe_window_is_binary_and_include_keeps_bytes():
+    from patchling.relevance import _prepare_content, is_binary_content
+
+    payload = b"a" * 8192 + b"\xff" + b"TAIL"
+    assert is_binary_content(payload) is True
+    assert _prepare_content(payload) == (True, "")
+
+    # A character split by the 8192 window is still text when the whole file is valid.
+    split = b"a" * 8191 + "é".encode("utf-8") + b"TAIL"
+    assert len(split) > 8192
+    binary, text = _prepare_content(split)
+    assert binary is False
+    assert text.endswith("éTAIL")
+    assert "\ufffd" not in text
+
+    calls = {"n": 0}
+
+    def fake_call_llm(api_key, base_url, model, messages, max_tokens, temperature, budget_tokens=None):
+        calls["n"] += 1
+        user = messages[-1]["content"]
+        assert "TAIL" not in user
+        return _Response("yes")
+
+    excluded = filter_relevant(
+        {"a.bin": payload},
+        "task",
+        model="m",
+        cache=False,
+        api_key="k",
+        on_binary="exclude",
+        call_llm=fake_call_llm,
+    )
+    assert calls["n"] == 0
+    assert excluded.files == {}
+    assert excluded.decisions[0].decision == "no"
+    assert excluded.decisions[0].skipped_reason == "binary"
+
+    included = filter_relevant(
+        {"a.bin": payload},
+        "task",
+        model="m",
+        cache=False,
+        api_key="k",
+        on_binary="include",
+        call_llm=fake_call_llm,
+    )
+    assert calls["n"] == 0
+    assert included.decisions[0].decision == "yes"
+    assert included.files["a.bin"] is payload
+    assert included.files["a.bin"].endswith(b"\xffTAIL")
+
+
+def test_leading_bom_is_preserved_in_decoded_text():
+    from patchling.relevance import PROMPT_VERSION, _prepare_content, _truncate_text
+
+    raw = b"\xef\xbb\xbfhello"
+    binary, text = _prepare_content(raw)
+    assert binary is False
+    assert text == "\ufeffhello"
+    assert text.startswith("\ufeff")
+    assert _truncate_text(text, None, 1, None).startswith("\ufeff")
+
+    with_bom = RelevanceCache.make_key(
+        model="m",
+        task="task",
+        content=text,
+        path="a.py",
+        max_output_tokens=1024,
+        prompt_version=PROMPT_VERSION,
+        effort="low",
+        mode="full",
+        note="",
+    )
+    without_bom = RelevanceCache.make_key(
+        model="m",
+        task="task",
+        content="hello",
+        path="a.py",
+        max_output_tokens=1024,
+        prompt_version=PROMPT_VERSION,
+        effort="low",
+        mode="full",
+        note="",
+    )
+    assert with_bom != without_bom
+
+    def fake_call_llm(api_key, base_url, model, messages, max_tokens, temperature, budget_tokens=None):
+        assert "\ufeffhello" in messages[-1]["content"]
+        return _Response("yes")
+
+    result = filter_relevant(
+        {"a.py": raw},
+        "task",
+        model="m",
+        cache=False,
+        api_key="k",
+        call_llm=fake_call_llm,
+    )
+    assert result.files["a.py"] == "\ufeffhello"
+
+
+def test_non_string_message_content_fails_open_uncached():
+    class Message:
+        content = ["no"]
+
+    class Choice:
+        message = Message()
+
+    class Response:
+        choices = [Choice()]
+        usage = None
+
+    payloads = [
+        {"choices": [{"message": {"content": ["no"]}}]},
+        Response(),
+    ]
+    for payload in payloads:
+        calls = {"n": 0}
+
+        def fake_call_llm(api_key, base_url, model, messages, max_tokens, temperature, budget_tokens=None, payload=payload):
+            calls["n"] += 1
+            return payload
+
+        cache = RelevanceCache()
+        result = filter_relevant(
+            {"a.py": "x\n"},
+            "task",
+            model="m",
+            cache=cache,
+            api_key="k",
+            call_llm=fake_call_llm,
+        )
+        decision = result.decisions[0]
+        assert decision.failed_open is True
+        assert decision.decision == "yes"
+        assert decision.cached is False
+        assert "malformed response" in (decision.error or "")
+        assert cache.get(decision.cache_key) is None if decision.cache_key else True
+        again = filter_relevant(
+            {"a.py": "x\n"},
+            "task",
+            model="m",
+            cache=cache,
+            api_key="k",
+            call_llm=fake_call_llm,
+        )
+        assert again.decisions[0].cached is False
+        assert calls["n"] == 2
+
+
+def test_default_wrapper_caches_empty_effort_when_legacy_sdk_rejects_it(monkeypatch):
+    """The real call_llm retries a legacy SDK; the cache key uses effort ""."""
+    from patchling.relevance import PROMPT_VERSION
+
+    calls = []
+
+    class Message:
+        content = "no, unrelated"
+
+    class Choice:
+        message = Message()
+
+    class Usage:
+        prompt_tokens = 3
+        completion_tokens = 2
+        total_tokens = 5
+
+    class Response:
+        choices = [Choice()]
+        usage = Usage()
+
+    class Completions:
+        def create(self, **kwargs):
+            calls.append("reasoning_effort" in kwargs)
+            if "reasoning_effort" in kwargs:
+                raise TypeError("unexpected keyword argument 'reasoning_effort'")
+            return Response()
+
+    class Chat:
+        completions = Completions()
+
+    class Client:
+        def __init__(self, **kwargs):
+            self.chat = Chat()
+
+    monkeypatch.setattr("patchling.core.OpenAI", Client)
+    cache = RelevanceCache()
+    files = {"a.py": "x = 1\n"}
+    first = filter_relevant(files, "task", model="m", cache=cache, api_key="k")
+    second = filter_relevant(files, "task", model="m", cache=cache, api_key="k")
+    assert calls == [True, False]
+    assert first.decisions[0].cached is False
+    assert first.decisions[0].decision == "no"
+    assert second.decisions[0].cached is True
+    assert second.decisions[0].decision == "no"
+    common = dict(
+        model="m",
+        task="task",
+        content="x = 1\n",
+        path="a.py",
+        max_output_tokens=1024,
+        prompt_version=PROMPT_VERSION,
+        mode="full",
+        note="",
+    )
+    low = RelevanceCache.make_key(**common, effort="low")
+    omitted = RelevanceCache.make_key(**common, effort="")
+    assert cache.get(low) is None
+    assert cache.get(omitted)["decision"] == "no"
+
+
 def test_envelope_errors_fail_open_and_are_not_cached():
     payloads = [
         {"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}},
@@ -1322,6 +1540,7 @@ def test_envelope_errors_fail_open_and_are_not_cached():
         {"choices": [{"message": {}}]},
         "this is not json",
         None,
+        {"choices": [{"message": {"content": ["no"]}}]},
     ]
     for payload in payloads:
         calls = {"n": 0}
@@ -1492,8 +1711,8 @@ def test_cli_apply_uses_the_full_original_file_map(tmp_path, monkeypatch):
 
 # sha256 of tests/fixtures/parity/relevance.json. That file is a byte-identical
 # copy of test/fixtures/parity/relevance.json on 255BITS/patchling branch
-# cursor/relevance-filter-915b. A mismatch fails CI.
-RELEVANCE_FIXTURE_SHA256 = "90b7299375518fbc6cb9bfeacb536f46f95c17ed72f8f66d063592bfdaec4665"
+# cursor/relevance-filter-followup-915b. A mismatch fails CI.
+RELEVANCE_FIXTURE_SHA256 = "6976ad71714be0138a6043d9bdeb724abe34a65515f8389913dca10a0c48c4f2"
 
 
 def test_vendored_relevance_fixture_matches_library():
@@ -1625,6 +1844,19 @@ def test_vendored_relevance_fixture_matches_library():
         expect = case.get("binary", case.get("expect_binary"))
         assert is_binary_content(sample) is expect, case
 
+    bom = data.get("bom")
+    if bom:
+        from patchling.relevance import _prepare_content
+
+        sample = bytes(bom["bytes"])
+        binary, text = _prepare_content(sample)
+        assert binary is False
+        assert text == bom["text"]
+        assert text.startswith("\ufeff")
+        assert count_tokens(text) == bom["tokens"]
+        assert _truncate_text(text, None, bom["maxTokens"], None) == bom["truncated"]
+        assert _truncate_text(text, None, bom["maxTokens"], None).startswith("\ufeff")
+
     ignore = data["ignore"]
     assert list(DEFAULT_SKIP_PATTERNS) == list(ignore["patterns"])
     for example in ignore["examples"]: