python-codex 0.2.6__py3-none-any.whl → 0.3.0__py3-none-any.whl

responses_server/stream_router.py

@@ -1,25 +1,20 @@
-
-import json
 import http.client
+import json
 import ssl
+import typing
 import urllib.error
 import urllib.request
 
 from .config import CompatServerConfig
-from .messages_api import (
-    MessagesAPIAdapterError,
-    build_messages_request,
-    iter_chat_chunks as iter_chat_chunks_from_messages,
-    saw_message_stop as messages_saw_message_stop,
-)
+from .messages_api import MessagesAPIAdapterError, build_messages_request
+from .messages_api import iter_chat_chunks as iter_chat_chunks_from_messages
+from .messages_api import saw_message_stop as messages_saw_message_stop
 from .session_store import StoredResponse
 from .tools import WebSearchTool, collect_custom_tool_names
-from .tools.custom_adapter import (
-    CustomToolAdapterError,
-    build_output_item as build_custom_output_item,
-    build_tool_call as build_custom_tool_call,
-    build_tool_definition as build_custom_tool_definition,
-)
+from .tools.custom_adapter import CustomToolAdapterError
+from .tools.custom_adapter import build_output_item as build_custom_output_item
+from .tools.custom_adapter import build_tool_call as build_custom_tool_call
+from .tools.custom_adapter import build_tool_definition as build_custom_tool_definition
 from .tools.web_search import (
     build_followup_request,
     build_output_items,
@@ -28,7 +23,6 @@ from .tools.web_search import (
     partition_tool_calls,
 )
 from .trajectory_dump import TrajectoryDumpWriter
-import typing
 
 
 class UnsupportedIncommingFeature(ValueError):
@@ -38,23 +32,23 @@ class UnsupportedIncommingFeature(ValueError):
 class OutcommingChatError(RuntimeError):
     def __init__(
         self,
-        message: 'str',
-        error_type: 'typing.Union[str, None]' = None,
-    ) -> 'None':
+        message: "str",
+        error_type: "typing.Union[str, None]" = None,
+    ) -> "None":
         super().__init__(message)
         self.error_type = error_type
 
 
 class StreamRouter:
-    def __init__(self, config: 'CompatServerConfig') -> 'None':
+    def __init__(self, config: "CompatServerConfig") -> "None":
         self._config = config
         self._mock_web_search = WebSearchTool()
 
     def _provider_capability(
         self,
-        explicit_support: 'typing.Dict[str, bool]',
-        default: 'typing.Union[bool, None]' = None,
-    ) -> 'bool':
+        explicit_support: "typing.Dict[str, bool]",
+        default: "typing.Union[bool, None]" = None,
+    ) -> "bool":
         provider_name = str(self._config.model_provider or "").strip().lower()
         if provider_name in explicit_support:
             return explicit_support[provider_name]
@@ -64,7 +58,7 @@ class StreamRouter:
             return default
         raise KeyError("provider capability map is missing `vllm` fallback")
 
-    def _supports_chat_reasoning(self) -> 'bool':
+    def _supports_chat_reasoning(self) -> "bool":
         # Unknown providers inherit the vLLM compatibility behavior unless a
         # provider is explicitly declared otherwise.
         return self._provider_capability(
@@ -74,7 +68,7 @@ class StreamRouter:
             }
         )
 
-    def _supports_stream_usage(self) -> 'bool':
+    def _supports_stream_usage(self) -> "bool":
         return self._provider_capability(
             {
                 "vllm": True,
@@ -84,8 +78,8 @@ class StreamRouter:
 
     def validate_incomming_request(
         self,
-        incomming_request: 'typing.Dict[str, object]',
-    ) -> 'None':
+        incomming_request: "typing.Dict[str, object]",
+    ) -> "None":
         model = str(incomming_request.get("model", "")).strip()
         if not model:
             raise UnsupportedIncommingFeature("incomming request is missing `model`")
@@ -104,11 +98,11 @@ class StreamRouter:
 
     def collect_custom_tool_names(
         self,
-        incomming_request: 'typing.Dict[str, object]',
-    ) -> 'typing.Set[str]':
+        incomming_request: "typing.Dict[str, object]",
+    ) -> "typing.Set[str]":
         return collect_custom_tool_names(incomming_request.get("tools") or [])
 
-    def list_models(self) -> 'typing.Dict[str, object]':
+    def list_models(self) -> "typing.Dict[str, object]":
         request = urllib.request.Request(
             self._config.outcomming_models_url(),
             headers=self._build_headers(accept="application/json"),
@@ -118,8 +112,8 @@ class StreamRouter:
 
     def build_outcomming_request(
         self,
-        incomming_request: 'typing.Dict[str, object]',
-    ) -> 'typing.Dict[str, object]':
+        incomming_request: "typing.Dict[str, object]",
+    ) -> "typing.Dict[str, object]":
         model = str(incomming_request.get("model", "")).strip()
         if not model:
             raise UnsupportedIncommingFeature("incomming request is missing `model`")
@@ -135,7 +129,7 @@ class StreamRouter:
         if not isinstance(input_items, list):
             raise UnsupportedIncommingFeature("incomming `input` must be a list")
 
-        payload: 'typing.Dict[str, object]' = {
+        payload: "typing.Dict[str, object]" = {
             "model": model,
             "messages": self._responses_input_to_chat_messages(
                 instructions,
@@ -143,6 +137,17 @@ class StreamRouter:
             ),
             "stream": True,
         }
+        reasoning = incomming_request.get("reasoning")
+        if isinstance(reasoning, dict):
+            reasoning_effort = reasoning.get("effort")
+            if isinstance(reasoning_effort, str) and reasoning_effort:
+                # vLLM's Responses renderer consumes effort as a template
+                # kwarg; keeping it out of the chat schema also preserves
+                # provider-specific values such as `max`.
+                payload["chat_template_kwargs"] = {
+                    "reasoning_effort": reasoning_effort,
+                }
+
         max_tokens = self._coerce_positive_int(
             incomming_request.get("max_output_tokens")
         )
@@ -169,7 +174,7 @@ class StreamRouter:
 
         return payload
 
-    def open_outcomming_stream(self, outcomming_request: 'typing.Dict[str, object]'):
+    def open_outcomming_stream(self, outcomming_request: "typing.Dict[str, object]"):
         outcomming_api = self._config.normalized_outcomming_api()
         if outcomming_api == "messages":
             return self._open_outcomming_messages_stream(outcomming_request)
@@ -181,7 +186,7 @@ class StreamRouter:
 
     def _open_outcomming_chat_stream(
         self,
-        outcomming_request: 'typing.Dict[str, object]',
+        outcomming_request: "typing.Dict[str, object]",
     ):
         request = urllib.request.Request(
             self._config.outcomming_chat_completions_url(),
@@ -203,7 +208,13 @@ class StreamRouter:
                         if data == "[DONE]":
                             saw_done = True
                             break
-                        yield json.loads(data)
+                        payload = json.loads(data)
+                        if isinstance(payload, dict) and payload.get("error") is not None:
+                            raise OutcommingChatError(
+                                "outcomming chat stream returned an error: "
+                                + json.dumps(payload["error"], ensure_ascii=False)[:500]
+                            )
+                        yield payload
                     if not saw_done:
                         raise OutcommingChatError(
                             "outcomming chat stream ended before [DONE]"
@@ -224,6 +235,11 @@ class StreamRouter:
             raise OutcommingChatError(
                 f"outcomming chat request failed with status {exc.code}: {body[:500]}"
             ) from exc
+        except TimeoutError as exc:
+            raise OutcommingChatError(
+                "outcomming chat request timed out after "
+                f"{self._config.timeout_seconds:g}s"
+            ) from exc
         except urllib.error.URLError as exc:
             raise OutcommingChatError(
                 f"outcomming chat request failed: {exc.reason}"
@@ -231,7 +247,7 @@ class StreamRouter:
 
     def _open_outcomming_messages_stream(
         self,
-        outcomming_request: 'typing.Dict[str, object]',
+        outcomming_request: "typing.Dict[str, object]",
     ):
         try:
             messages_request = build_messages_request(outcomming_request)
@@ -251,7 +267,7 @@ class StreamRouter:
                 timeout=self._config.timeout_seconds,
             ) as response:
                 try:
-                    stream_state: 'typing.Dict[str, object]' = {}
+                    stream_state: "typing.Dict[str, object]" = {}
                     for event_name, data in self._iter_sse_events(response):
                         if not data:
                             continue
@@ -285,6 +301,11 @@ class StreamRouter:
             raise OutcommingChatError(
                 f"outcomming messages request failed with status {exc.code}: {body[:500]}"
             ) from exc
+        except TimeoutError as exc:
+            raise OutcommingChatError(
+                "outcomming messages request timed out after "
+                f"{self._config.timeout_seconds:g}s"
+            ) from exc
         except urllib.error.URLError as exc:
             raise OutcommingChatError(
                 f"outcomming messages request failed: {exc.reason}"
@@ -292,10 +313,10 @@ class StreamRouter:
 
     def route_stream(
         self,
-        stored_response: 'StoredResponse',
-        outcomming_request: 'typing.Dict[str, object]',
-        custom_tool_names: 'typing.Union[typing.Set[str], None]' = None,
-        trajectory_dump: 'typing.Union[TrajectoryDumpWriter, None]' = None,
+        stored_response: "StoredResponse",
+        outcomming_request: "typing.Dict[str, object]",
+        custom_tool_names: "typing.Union[typing.Set[str], None]" = None,
+        trajectory_dump: "typing.Union[TrajectoryDumpWriter, None]" = None,
     ):
         yield (
             "response.created",
@@ -310,9 +331,9 @@ class StreamRouter:
             },
         )
 
-        text_parts: 'typing.List[str]' = []
-        reasoning_parts: 'typing.List[str]' = []
-        latest_usage: 'typing.Dict[str, object]' = {}
+        text_parts: "typing.List[str]" = []
+        reasoning_parts: "typing.List[str]" = []
+        latest_usage: "typing.Dict[str, object]" = {}
         current_request = json.loads(json.dumps(outcomming_request))
         current_stream = self._open_tracked_outcomming_stream(
             current_request,
@@ -321,9 +342,9 @@ class StreamRouter:
         retried_reasoning_only_output = False
 
         while True:
-            tool_calls: 'typing.Dict[int, typing.Dict[str, object]]' = {}
-            finish_reasons: 'typing.List[str]' = []
-            current_usage: 'typing.Dict[str, object]' = {}
+            tool_calls: "typing.Dict[int, typing.Dict[str, object]]" = {}
+            finish_reasons: "typing.List[str]" = []
+            current_usage: "typing.Dict[str, object]" = {}
             reasoning_start = len(reasoning_parts)
             text_start = len(text_parts)
             for chunk in current_stream:
@@ -437,26 +458,30 @@ class StreamRouter:
 
     def _open_tracked_outcomming_stream(
         self,
-        outcomming_request: 'typing.Dict[str, object]',
-        trajectory_dump: 'typing.Union[TrajectoryDumpWriter, None]' = None,
+        outcomming_request: "typing.Dict[str, object]",
+        trajectory_dump: "typing.Union[TrajectoryDumpWriter, None]" = None,
     ):
         outcomming_stream = self.open_outcomming_stream(outcomming_request)
         if trajectory_dump is None:
             return outcomming_stream
-        return trajectory_dump.wrap_stream(outcomming_stream)
+        return trajectory_dump.wrap_stream(
+            outcomming_stream,
+            outcomming_request,
+        )
 
     def _responses_input_to_chat_messages(
         self,
-        instructions: 'str',
-        input_items: 'typing.List[object]',
-    ) -> 'typing.List[typing.Dict[str, object]]':
-        messages: 'typing.List[typing.Dict[str, object]]' = []
+        instructions: "str",
+        input_items: "typing.List[object]",
+    ) -> "typing.List[typing.Dict[str, object]]":
+        messages: "typing.List[typing.Dict[str, object]]" = []
         if instructions:
             messages.append({"role": "developer", "content": instructions})
 
-        pending_assistant: 'typing.Union[typing.Dict[str, object], None]' = None
+        pending_assistant: "typing.Union[typing.Dict[str, object], None]" = None
+        pending_tool_images: "typing.List[typing.Dict[str, object]]" = []
 
-        def flush_pending_assistant() -> 'None':
+        def flush_pending_assistant() -> "None":
             nonlocal pending_assistant
             if pending_assistant is None:
                 return
@@ -470,12 +495,20 @@ class StreamRouter:
             messages.append(pending_assistant)
             pending_assistant = None
 
+        def flush_pending_tool_images() -> "None":
+            if not pending_tool_images:
+                return
+            messages.append({"role": "user", "content": list(pending_tool_images)})
+            del pending_tool_images[:]
+
         for raw_item in input_items:
             if not isinstance(raw_item, dict):
                 raise UnsupportedIncommingFeature(
                     "all incomming `input` items must be objects"
                 )
             item_type = raw_item.get("type")
+            if item_type not in {"function_call_output", "custom_tool_call_output"}:
+                flush_pending_tool_images()
 
             if item_type == "message":
                 role = str(raw_item.get("role", "")).strip()
@@ -483,8 +516,12 @@ class StreamRouter:
                     raise UnsupportedIncommingFeature(
                         f"unsupported incomming message role: {role or '<empty>'}"
                     )
-                text = self._coalesce_content_text(raw_item.get("content"))
+                text, image_parts = self._split_content_parts(raw_item.get("content"))
                 if role == "assistant":
+                    if image_parts:
+                        raise UnsupportedIncommingFeature(
+                            "assistant messages cannot carry `input_image` content parts"
+                        )
                     if pending_assistant is None:
                         pending_assistant = {"role": "assistant"}
                     if text:
@@ -493,7 +530,13 @@ class StreamRouter:
                         )
                     continue
                 flush_pending_assistant()
-                messages.append({"role": role, "content": text})
+                if image_parts:
+                    content: "object" = (
+                        [{"type": "text", "text": text}] if text else []
+                    ) + image_parts
+                else:
+                    content = text
+                messages.append({"role": role, "content": content})
                 continue
 
             if item_type == "reasoning":
@@ -532,15 +575,17 @@ class StreamRouter:
 
             if item_type == "function_call_output":
                 flush_pending_assistant()
+                text, image_parts = self._split_tool_output_parts(
+                    raw_item.get("output")
+                )
                 messages.append(
                     {
                         "role": "tool",
                         "tool_call_id": str(raw_item.get("call_id", "")).strip(),
-                        "content": self._coalesce_tool_output_text(
-                            raw_item.get("output")
-                        ),
+                        "content": text,
                     }
                 )
+                pending_tool_images.extend(image_parts)
                 continue
 
             if item_type == "custom_tool_call":
@@ -559,15 +604,17 @@ class StreamRouter:
 
             if item_type == "custom_tool_call_output":
                 flush_pending_assistant()
+                text, image_parts = self._split_tool_output_parts(
+                    raw_item.get("output")
+                )
                 messages.append(
                     {
                         "role": "tool",
                         "tool_call_id": str(raw_item.get("call_id", "")).strip(),
-                        "content": self._coalesce_tool_output_text(
-                            raw_item.get("output")
-                        ),
+                        "content": text,
                     }
                 )
+                pending_tool_images.extend(image_parts)
                 continue
 
             raise UnsupportedIncommingFeature(
@@ -575,26 +622,31 @@ class StreamRouter:
             )
 
         flush_pending_assistant()
+        flush_pending_tool_images()
         return messages
 
-    def _coerce_positive_int(self, raw_value: 'object') -> 'typing.Union[int, None]':
+    def _coerce_positive_int(self, raw_value: "object") -> "typing.Union[int, None]":
         if isinstance(raw_value, bool):
             return None
         if isinstance(raw_value, int) and raw_value > 0:
             return raw_value
         return None
 
-    def _coalesce_content_text(self, raw_content: 'object') -> 'str':
+    def _split_content_parts(
+        self,
+        raw_content: "object",
+    ) -> "typing.Tuple[str, typing.List[typing.Dict[str, object]]]":
         if raw_content is None:
-            return ""
+            return "", []
         if isinstance(raw_content, str):
-            return raw_content
+            return raw_content, []
         if not isinstance(raw_content, list):
             raise UnsupportedIncommingFeature(
                 "message `content` must be a list or string"
             )
 
-        text_parts: 'typing.List[str]' = []
+        text_parts: "typing.List[str]" = []
+        image_parts: "typing.List[typing.Dict[str, object]]" = []
         for part in raw_content:
             if not isinstance(part, dict):
                 raise UnsupportedIncommingFeature(
@@ -604,22 +656,43 @@ class StreamRouter:
             if part_type in {"input_text", "output_text"}:
                 text_parts.append(str(part.get("text", "")))
                 continue
+            if part_type == "input_image":
+                image_parts.append(self._build_chat_image_part(part))
+                continue
             raise UnsupportedIncommingFeature(
                 f"content part type `{part_type}` is not yet supported by the chat backend"
             )
-        return "".join(text_parts)
+        return "".join(text_parts), image_parts
+
+    def _build_chat_image_part(
+        self,
+        part: "typing.Dict[str, object]",
+    ) -> "typing.Dict[str, object]":
+        image_url = str(part.get("image_url", "") or "").strip()
+        if not image_url:
+            raise UnsupportedIncommingFeature(
+                "`input_image` content parts must carry a non-empty `image_url`"
+            )
+        image_payload: "typing.Dict[str, object]" = {"url": image_url}
+        detail = part.get("detail")
+        if isinstance(detail, str) and detail in {"auto", "low", "high"}:
+            image_payload["detail"] = detail
+        return {"type": "image_url", "image_url": image_payload}
 
-    def _coalesce_tool_output_text(self, raw_output: 'object') -> 'str':
+    def _split_tool_output_parts(
+        self,
+        raw_output: "object",
+    ) -> "typing.Tuple[str, typing.List[typing.Dict[str, object]]]":
         if isinstance(raw_output, str):
-            return raw_output
+            return raw_output, []
         if isinstance(raw_output, list):
-            return self._coalesce_content_text(raw_output)
-        return json.dumps(raw_output, ensure_ascii=False)
+            return self._split_content_parts(raw_output)
+        return json.dumps(raw_output, ensure_ascii=False), []
 
-    def _coalesce_reasoning_text(self, raw_item: 'typing.Dict[str, object]') -> 'str':
+    def _coalesce_reasoning_text(self, raw_item: "typing.Dict[str, object]") -> "str":
         content = raw_item.get("content")
         if isinstance(content, list):
-            text_parts: 'typing.List[str]' = []
+            text_parts: "typing.List[str]" = []
             for part in content:
                 if not isinstance(part, dict):
                     continue
@@ -646,8 +719,10 @@ class StreamRouter:
                 return value
         return ""
 
-    def _translate_tools(self, incomming_tools: 'typing.List[object]') -> 'typing.List[typing.Dict[str, object]]':
-        translated: 'typing.List[typing.Dict[str, object]]' = []
+    def _translate_tools(
+        self, incomming_tools: "typing.List[object]"
+    ) -> "typing.List[typing.Dict[str, object]]":
+        translated: "typing.List[typing.Dict[str, object]]" = []
         for raw_tool in incomming_tools:
             if not isinstance(raw_tool, dict):
                 raise UnsupportedIncommingFeature("tool definitions must be objects")
@@ -682,7 +757,7 @@ class StreamRouter:
             )
         return translated
 
-    def _translate_tool_choice(self, raw_tool_choice: 'object') -> 'object':
+    def _translate_tool_choice(self, raw_tool_choice: "object") -> "object":
         if isinstance(raw_tool_choice, str):
             return raw_tool_choice
         if not isinstance(raw_tool_choice, dict):
@@ -705,14 +780,14 @@ class StreamRouter:
 
     def _consume_chat_chunk(
         self,
-        payload: 'typing.Dict[str, object]',
-        reasoning_parts: 'typing.List[str]',
-        text_parts: 'typing.List[str]',
-        tool_calls: 'typing.Dict[int, typing.Dict[str, object]]',
-        current_usage: 'typing.Dict[str, object]',
-        finish_reasons: 'typing.List[str]',
-    ) -> 'typing.List[typing.Tuple[str, typing.Dict[str, object]]]':
-        events: 'typing.List[typing.Tuple[str, typing.Dict[str, object]]]' = []
+        payload: "typing.Dict[str, object]",
+        reasoning_parts: "typing.List[str]",
+        text_parts: "typing.List[str]",
+        tool_calls: "typing.Dict[int, typing.Dict[str, object]]",
+        current_usage: "typing.Dict[str, object]",
+        finish_reasons: "typing.List[str]",
+    ) -> "typing.List[typing.Tuple[str, typing.Dict[str, object]]]":
+        events: "typing.List[typing.Tuple[str, typing.Dict[str, object]]]" = []
         usage = payload.get("usage")
         if isinstance(usage, dict):
             self._capture_usage_snapshot(current_usage, usage)
@@ -732,12 +807,20 @@ class StreamRouter:
                 continue
 
             reasoning = delta.get("reasoning")
-            if isinstance(reasoning, str) and reasoning:
-                reasoning_parts.append(reasoning)
-
             reasoning_content = delta.get("reasoning_content")
-            if isinstance(reasoning_content, str) and reasoning_content:
-                reasoning_parts.append(reasoning_content)
+            if (
+                isinstance(reasoning, str)
+                and reasoning
+                and isinstance(reasoning_content, str)
+                and reasoning == reasoning_content
+            ):
+                # Some chat providers expose the same delta under both aliases.
+                reasoning_parts.append(reasoning)
+            else:
+                if isinstance(reasoning, str) and reasoning:
+                    reasoning_parts.append(reasoning)
+                if isinstance(reasoning_content, str) and reasoning_content:
+                    reasoning_parts.append(reasoning_content)
 
             content = delta.get("content")
             if isinstance(content, str) and content:
@@ -795,9 +878,9 @@ class StreamRouter:
 
     def _capture_usage_snapshot(
         self,
-        current_usage: 'typing.Dict[str, object]',
-        usage: 'typing.Dict[str, object]',
-    ) -> 'None':
+        current_usage: "typing.Dict[str, object]",
+        usage: "typing.Dict[str, object]",
+    ) -> "None":
         scalar_mappings = (
             ("input_tokens", usage.get("input_tokens", usage.get("prompt_tokens"))),
             (
@@ -829,12 +912,12 @@ class StreamRouter:
 
     def _build_output_items(
         self,
-        reasoning_parts: 'typing.List[str]',
-        text_parts: 'typing.List[str]',
-        tool_calls: 'typing.Dict[int, typing.Dict[str, object]]',
-        custom_tool_names: 'typing.Set[str]',
-    ) -> 'typing.List[typing.Dict[str, object]]':
-        items: 'typing.List[typing.Dict[str, object]]' = []
+        reasoning_parts: "typing.List[str]",
+        text_parts: "typing.List[str]",
+        tool_calls: "typing.Dict[int, typing.Dict[str, object]]",
+        custom_tool_names: "typing.Set[str]",
+    ) -> "typing.List[typing.Dict[str, object]]":
+        items: "typing.List[typing.Dict[str, object]]" = []
         reasoning_text = "".join(reasoning_parts)
         if reasoning_text:
             items.append(
@@ -890,8 +973,7 @@ class StreamRouter:
             items.append(
                 {
                     "type": "function_call",
-                    "call_id": str(tool_call.get("id", "")).strip()
-                    or f"call_{index}",
+                    "call_id": str(tool_call.get("id", "")).strip() or f"call_{index}",
                     "name": name,
                     "arguments": arguments,
                 }
@@ -899,7 +981,9 @@ class StreamRouter:
 
         return items
 
-    def _request_json(self, request: 'urllib.request.Request') -> 'typing.Dict[str, object]':
+    def _request_json(
+        self, request: "urllib.request.Request"
+    ) -> "typing.Dict[str, object]":
         try:
             with urllib.request.urlopen(
                 request,
@@ -912,12 +996,17 @@ class StreamRouter:
             raise OutcommingChatError(
                 f"outcomming request failed with status {exc.code}: {body[:500]}"
             ) from exc
+        except TimeoutError as exc:
+            raise OutcommingChatError(
+                "outcomming request timed out after "
+                f"{self._config.timeout_seconds:g}s"
+            ) from exc
         except urllib.error.URLError as exc:
             raise OutcommingChatError(
                 f"outcomming request failed: {exc.reason}"
             ) from exc
 
-    def _build_headers(self, accept: 'str') -> 'typing.Dict[str, str]':
+    def _build_headers(self, accept: "str") -> "typing.Dict[str, str]":
         headers = {
             "Accept": accept,
             "Content-Type": "application/json",
@@ -928,8 +1017,8 @@ class StreamRouter:
         return headers
 
     def _iter_sse_events(self, response):
-        event_name: 'typing.Union[str, None]' = None
-        data_lines: 'typing.List[str]' = []
+        event_name: "typing.Union[str, None]" = None
+        data_lines: "typing.List[str]" = []
 
         for raw_line in response:
             line = raw_line.decode("utf-8", errors="replace").rstrip("\r\n")