json-repair 0.43.0__py3-none-any.whl → 0.44.1__py3-none-any.whl

json_repair/json_parser.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Literal, Optional, TextIO, Tuple, Union
+from typing import Any, ClassVar, Dict, List, Literal, Optional, TextIO, Tuple, Union
 
 from .json_context import ContextValues, JsonContext
 from .object_comparer import ObjectComparer
@@ -9,7 +9,7 @@ JSONReturnType = Union[Dict[str, Any], List[Any], str, float, int, bool, None]
 
 class JSONParser:
     # Constants
-    STRING_DELIMITERS = ['"', "'", "“", "”"]
+    STRING_DELIMITERS: ClassVar[list[str]] = ['"', "'", "“", "”"]
 
     def __init__(
         self,
@@ -17,6 +17,7 @@ class JSONParser:
         json_fd: Optional[TextIO],
         logging: Optional[bool],
         json_fd_chunk_length: int = 0,
+        stream_stable: bool = False,
     ) -> None:
         # The string to parse
         self.json_str: Union[str, StringFileWrapper] = json_str
@@ -40,6 +41,14 @@ class JSONParser:
         else:
             # No-op
             self.log = lambda *args, **kwargs: None
+        # When the json to be repaired is the accumulation of streaming json at a certain moment.
+        # e.g. json obtained from llm response.
+        # If this parameter to True will keep the repair results stable. For example:
+        #   case 1:  '{"key": "val\\' => '{"key": "val"}'
+        #   case 2:  '{"key": "val\\n' => '{"key": "val\\n"}'
+        #   case 3:  '{"key": "val\\n123,`key2:value2' => '{"key": "val\\n123,`key2:value2"}'
+        #   case 4:  '{"key": "val\\n123,`key2:value2`"}' => '{"key": "val\\n123,`key2:value2`"}'
+        self.stream_stable = stream_stable
 
     def parse(
         self,
@@ -159,6 +168,10 @@ class JSONParser:
                                     and isinstance(new_array[0], list)
                                     else new_array
                                 )
+                            self.skip_whitespaces_at()
+                            if self.get_char_at() == ",":
+                                self.index += 1
+                            self.skip_whitespaces_at()
                             continue
                         else:
                             self.index = rollback_index
@@ -330,7 +343,7 @@ class JSONParser:
                     # Ok this is not a doubled quote, check if this is an empty string or not
                     i = self.skip_whitespaces_at(idx=1, move_main_index=False)
                     next_c = self.get_char_at(i)
-                    if next_c in self.STRING_DELIMITERS + ["{", "["]:
+                    if next_c in [*self.STRING_DELIMITERS, "{", "["]:
                         # something fishy is going on here
                         self.log(
                             "While parsing a string, we found a doubled quote but also another quote afterwards, ignoring it",
@@ -374,10 +387,15 @@ class JSONParser:
                     "While parsing a string missing the left delimiter in object key context, we found a :, stopping here",
                 )
                 break
-            if self.context.current == ContextValues.OBJECT_VALUE and char in [
-                ",",
-                "}",
-            ]:
+            if (
+                (missing_quotes or not self.stream_stable)
+                and self.context.current == ContextValues.OBJECT_VALUE
+                and char
+                in [
+                    ",",
+                    "}",
+                ]
+            ):
                 rstring_delimiter_missing = True
                 # check if this is a case in which the closing comma is NOT missing instead
                 i = self.skip_to_character(character=rstring_delimiter, idx=1)
@@ -446,7 +464,11 @@ class JSONParser:
                         "While parsing a string missing the left delimiter in object value context, we found a , or } and we couldn't determine that a right delimiter was present. Stopping here",
                     )
                     break
-            if char == "]" and ContextValues.ARRAY in self.context.context:
+            if (
+                (missing_quotes or not self.stream_stable)
+                and char == "]"
+                and ContextValues.ARRAY in self.context.context
+            ):
                 # We found the end of an array and we are in array context
                 # So let's check if we find a rstring_delimiter forward otherwise end early
                 i = self.skip_to_character(rstring_delimiter)
@@ -456,6 +478,9 @@ class JSONParser:
             string_acc += char
             self.index += 1
             char = self.get_char_at()
+            # Unclosed string ends with a \ character. This character is ignored if stream_stable = True.
+            if self.stream_stable and not char and string_acc[-1] == "\\":
+                string_acc = string_acc[:-1]
             if char and string_acc[-1] == "\\":
                 # This is a special case, if people use real strings this might happen
                 self.log("Found a stray escape sequence, normalizing it")
@@ -665,14 +690,18 @@ class JSONParser:
         # A fallout of the previous special case in the while loop,
         # we need to update the index only if we had a closing quote
         if char != rstring_delimiter:
-            self.log(
-                "While parsing a string, we missed the closing quote, ignoring",
-            )
-            string_acc = string_acc.rstrip()
+            # if stream_stable = True, unclosed strings do not trim trailing whitespace characters
+            if not self.stream_stable:
+                self.log(
+                    "While parsing a string, we missed the closing quote, ignoring",
+                )
+                string_acc = string_acc.rstrip()
         else:
             self.index += 1
 
-        if missing_quotes or (string_acc and string_acc[-1] == "\n"):
+        if not self.stream_stable and (
+            missing_quotes or (string_acc and string_acc[-1] == "\n")
+        ):
             # Clean the whitespaces for some corner cases
             string_acc = string_acc.rstrip()
 

json_repair/json_repair.py

@@ -25,11 +25,37 @@ All supported use cases are in the unit tests
 import argparse
 import json
 import sys
-from typing import Dict, List, Optional, TextIO, Tuple, Union
+from typing import Dict, List, Literal, Optional, TextIO, Tuple, Union, overload
 
 from .json_parser import JSONParser, JSONReturnType
 
 
+@overload
+def repair_json(
+    json_str: str = "",
+    return_objects: Literal[False] = False,
+    skip_json_loads: bool = False,
+    logging: bool = False,
+    json_fd: Optional[TextIO] = None,
+    ensure_ascii: bool = True,
+    chunk_length: int = 0,
+    stream_stable: bool = False,
+) -> str: ...
+
+
+@overload
+def repair_json(
+    json_str: str = "",
+    return_objects: Literal[True] = True,
+    skip_json_loads: bool = False,
+    logging: bool = False,
+    json_fd: Optional[TextIO] = None,
+    ensure_ascii: bool = True,
+    chunk_length: int = 0,
+    stream_stable: bool = False,
+) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: ...
+
+
 def repair_json(
     json_str: str = "",
     return_objects: bool = False,
@@ -38,6 +64,7 @@ def repair_json(
     json_fd: Optional[TextIO] = None,
     ensure_ascii: bool = True,
     chunk_length: int = 0,
+    stream_stable: bool = False,
 ) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
@@ -50,11 +77,11 @@ def repair_json(
         json_fd (Optional[TextIO], optional): File descriptor for JSON input. Do not use! Use `from_file` or `load` instead. Defaults to None.
         ensure_ascii (bool, optional): Set to False to avoid converting non-latin characters to ascii (for example when using chinese characters). Defaults to True. Ignored if `skip_json_loads` is True.
         chunk_length (int, optional): Size in bytes of the file chunks to read at once. Ignored if `json_fd` is None. Do not use! Use `from_file` or `load` instead. Defaults to 1MB.
-
+        stream_stable (bool, optional): When the json to be repaired is the accumulation of streaming json at a certain moment.If this parameter to True will keep the repair results stable.
     Returns:
         Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON or a tuple with the repaired JSON and repair log.
     """
-    parser = JSONParser(json_str, json_fd, logging, chunk_length)
+    parser = JSONParser(json_str, json_fd, logging, chunk_length, stream_stable)
     if skip_json_loads:
         parsed_json = parser.parse()
     else:
@@ -76,7 +103,8 @@ def loads(
     json_str: str,
     skip_json_loads: bool = False,
     logging: bool = False,
-) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
+    stream_stable: bool = False,
+) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]], str]:
     """
     This function works like `json.loads()` except that it will fix your JSON in the process.
     It is a wrapper around the `repair_json()` function with `return_objects=True`.
@@ -87,13 +115,14 @@ def loads(
         logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
 
     Returns:
-        Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
+        Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]], str]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
     """
     return repair_json(
         json_str=json_str,
         return_objects=True,
         skip_json_loads=skip_json_loads,
         logging=logging,
+        stream_stable=stream_stable,
     )
 
 
@@ -238,7 +267,7 @@ def cli(inline_args: Optional[List[str]] = None) -> int:
         else:
             print(json.dumps(result, indent=args.indent, ensure_ascii=ensure_ascii))
     except Exception as e:  # pragma: no cover
-        print(f"Error: {str(e)}", file=sys.stderr)
+        print(f"Error: {e!s}", file=sys.stderr)
         return 1
 
     return 0  # Success

{json_repair-0.43.0.dist-info → json_repair-0.44.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: json_repair
-Version: 0.43.0
+Version: 0.44.1
 Summary: A package to repair broken json strings
 Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
 License: MIT License
@@ -188,6 +188,14 @@ Some rules of thumb to use:
 - `skip_json_loads` is faster only if you 100% know that the string is not a valid JSON
 - If you are having issues with escaping pass the string as **raw** string like: `r"string with escaping\""`
 
+### Use json_repair with streaming
+
+Sometimes you are streaming some data and want to repair the JSON coming from it. Normally this won't work but you can pass `stream_stable` to `repair_json()` or `loads()` to make it work:
+
+```
+stream_output = repair_json(stream_input, stream_stable=True)
+```
+
 ### Use json_repair from CLI
 
 Install the library for command-line with:

{json_repair-0.43.0.dist-info → json_repair-0.44.1.dist-info}/RECORD

@@ -1,14 +1,14 @@
 json_repair/__init__.py,sha256=c4L2kZrHvWEKfj_ODU2naliNuvU6FlFVxtF0hbLe6s8,178
 json_repair/__main__.py,sha256=EsJb-y89uZEvGQQg1GdIDWzfDwfOMvVekKEtdguQXCM,67
 json_repair/json_context.py,sha256=mm6dOyrPJ1sDskTORZSXCW7W9-5veMlUKqXQ3Hw3EG4,971
-json_repair/json_parser.py,sha256=GFziN2KELYDWElzPda5wPfSeFIHYF8enJSr0c2YzKmQ,40451
-json_repair/json_repair.py,sha256=k-5HRRlCqrxNmJi0u1KE3IUeL4HXqi1XZ7oAL-NFDLo,10314
+json_repair/json_parser.py,sha256=wmDgXAroQ4gYZdi4Tbdn3LKXnx2x2v_uanzSzqP0aSQ,42003
+json_repair/json_repair.py,sha256=r-Mtr16U_n2wmHX_zNRZI2ZlLc0AV0fLWlLzGEWjJa0,11312
 json_repair/object_comparer.py,sha256=SeicB6_N4BHAEPon7s2BELEaJc4oyR9ZhfX2RgPk6Bw,1682
 json_repair/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 json_repair/string_file_wrapper.py,sha256=koZmdq2-Z5K7XF1bDqX6dEbNaVMJYcMTjq-aGe6NQvA,4526
-json_repair-0.43.0.dist-info/licenses/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
-json_repair-0.43.0.dist-info/METADATA,sha256=fR8K3LENGHxwkjG2pVRedVsh6XGBq0JWPV3weLz1KIM,11860
-json_repair-0.43.0.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
-json_repair-0.43.0.dist-info/entry_points.txt,sha256=SNfge3zPSP-ASqriYU9r3NAPaXdseYr7ciPMKdV2uSw,57
-json_repair-0.43.0.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
-json_repair-0.43.0.dist-info/RECORD,,
+json_repair-0.44.1.dist-info/licenses/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
+json_repair-0.44.1.dist-info/METADATA,sha256=VJv39wNseOemAA5tQe1dIW4ZPXFoLXFTPBCGLlLRwN8,12157
+json_repair-0.44.1.dist-info/WHEEL,sha256=ooBFpIzZCPdw3uqIQsOo4qqbA4ZRPxHnOH7peeONza0,91
+json_repair-0.44.1.dist-info/entry_points.txt,sha256=SNfge3zPSP-ASqriYU9r3NAPaXdseYr7ciPMKdV2uSw,57
+json_repair-0.44.1.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
+json_repair-0.44.1.dist-info/RECORD,,

{json_repair-0.43.0.dist-info → json_repair-0.44.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.0.0)
+Generator: setuptools (80.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any