json-repair 0.42.0__py3-none-any.whl → 0.44.0__py3-none-any.whl

json_repair/json_parser.py

@@ -17,6 +17,7 @@ class JSONParser:
         json_fd: Optional[TextIO],
         logging: Optional[bool],
         json_fd_chunk_length: int = 0,
+        stream_stable: bool = False,
     ) -> None:
         # The string to parse
         self.json_str: Union[str, StringFileWrapper] = json_str
@@ -40,6 +41,14 @@ class JSONParser:
         else:
             # No-op
             self.log = lambda *args, **kwargs: None
+        # When the json to be repaired is the accumulation of streaming json at a certain moment.
+        # e.g. json obtained from llm response.
+        # If this parameter to True will keep the repair results stable. For example:
+        #   case 1:  '{"key": "val\\' => '{"key": "val"}'
+        #   case 2:  '{"key": "val\\n' => '{"key": "val\\n"}'
+        #   case 3:  '{"key": "val\\n123,`key2:value2' => '{"key": "val\\n123,`key2:value2"}'
+        #   case 4:  '{"key": "val\\n123,`key2:value2`"}' => '{"key": "val\\n123,`key2:value2`"}'
+        self.stream_stable = stream_stable
 
     def parse(
         self,
@@ -113,7 +122,7 @@ class JSONParser:
 
     def parse_object(self) -> Dict[str, JSONReturnType]:
         # <object> ::= '{' [ <member> *(', ' <member>) ] '}' ; A sequence of 'members'
-        obj = {}
+        obj: Dict[str, JSONReturnType] = {}
         # Stop when you either find the closing parentheses or you have iterated over the entire string
         while (self.get_char_at() or "}") != "}":
             # This is what we expect to find:
@@ -141,6 +150,31 @@ class JSONParser:
             while self.get_char_at():
                 # The rollback index needs to be updated here in case the key is empty
                 rollback_index = self.index
+                if self.get_char_at() == "[" and key == "":
+                    # Is this an array?
+                    # Need to check if the previous parsed value contained in obj is an array and in that case parse and merge the two
+                    prev_key = list(obj.keys())[-1] if obj else None
+                    if prev_key and isinstance(obj[prev_key], list):
+                        # If the previous key's value is an array, parse the new array and merge
+                        self.index += 1
+                        new_array = self.parse_array()
+                        if isinstance(new_array, list):
+                            # Merge and flatten the arrays
+                            prev_value = obj[prev_key]
+                            if isinstance(prev_value, list):
+                                prev_value.extend(
+                                    new_array[0]
+                                    if len(new_array) == 1
+                                    and isinstance(new_array[0], list)
+                                    else new_array
+                                )
+                            self.skip_whitespaces_at()
+                            if self.get_char_at() == ",":
+                                self.index += 1
+                            self.skip_whitespaces_at()
+                            continue
+                        else:
+                            self.index = rollback_index
                 key = str(self.parse_string())
                 if key == "":
                     self.skip_whitespaces_at()
@@ -216,7 +250,7 @@ class JSONParser:
 
             # skip over whitespace after a value but before closing ]
             char = self.get_char_at()
-            while char and (char.isspace() or char == ","):
+            while char and char != "]" and (char.isspace() or char == ","):
                 self.index += 1
                 char = self.get_char_at()
 
@@ -353,10 +387,15 @@ class JSONParser:
                     "While parsing a string missing the left delimiter in object key context, we found a :, stopping here",
                 )
                 break
-            if self.context.current == ContextValues.OBJECT_VALUE and char in [
-                ",",
-                "}",
-            ]:
+            if (
+                (missing_quotes or not self.stream_stable)
+                and self.context.current == ContextValues.OBJECT_VALUE
+                and char
+                in [
+                    ",",
+                    "}",
+                ]
+            ):
                 rstring_delimiter_missing = True
                 # check if this is a case in which the closing comma is NOT missing instead
                 i = self.skip_to_character(character=rstring_delimiter, idx=1)
@@ -425,7 +464,11 @@ class JSONParser:
                         "While parsing a string missing the left delimiter in object value context, we found a , or } and we couldn't determine that a right delimiter was present. Stopping here",
                     )
                     break
-            if char == "]" and ContextValues.ARRAY in self.context.context:
+            if (
+                (missing_quotes or not self.stream_stable)
+                and char == "]"
+                and ContextValues.ARRAY in self.context.context
+            ):
                 # We found the end of an array and we are in array context
                 # So let's check if we find a rstring_delimiter forward otherwise end early
                 i = self.skip_to_character(rstring_delimiter)
@@ -435,6 +478,9 @@ class JSONParser:
             string_acc += char
             self.index += 1
             char = self.get_char_at()
+            # Unclosed string ends with a \ character. This character is ignored if stream_stable = True.
+            if self.stream_stable and not char and string_acc[-1] == "\\":
+                string_acc = string_acc[:-1]
             if char and string_acc[-1] == "\\":
                 # This is a special case, if people use real strings this might happen
                 self.log("Found a stray escape sequence, normalizing it")
@@ -644,14 +690,18 @@ class JSONParser:
         # A fallout of the previous special case in the while loop,
         # we need to update the index only if we had a closing quote
         if char != rstring_delimiter:
-            self.log(
-                "While parsing a string, we missed the closing quote, ignoring",
-            )
-            string_acc = string_acc.rstrip()
+            # if stream_stable = True, unclosed strings do not trim trailing whitespace characters
+            if not self.stream_stable:
+                self.log(
+                    "While parsing a string, we missed the closing quote, ignoring",
+                )
+                string_acc = string_acc.rstrip()
         else:
             self.index += 1
 
-        if missing_quotes or (string_acc and string_acc[-1] == "\n"):
+        if not self.stream_stable and (
+            missing_quotes or (string_acc and string_acc[-1] == "\n")
+        ):
             # Clean the whitespaces for some corner cases
             string_acc = string_acc.rstrip()
 

json_repair/json_repair.py

@@ -38,6 +38,7 @@ def repair_json(
     json_fd: Optional[TextIO] = None,
     ensure_ascii: bool = True,
     chunk_length: int = 0,
+    stream_stable: bool = False,
 ) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
@@ -50,11 +51,11 @@ def repair_json(
         json_fd (Optional[TextIO], optional): File descriptor for JSON input. Do not use! Use `from_file` or `load` instead. Defaults to None.
         ensure_ascii (bool, optional): Set to False to avoid converting non-latin characters to ascii (for example when using chinese characters). Defaults to True. Ignored if `skip_json_loads` is True.
         chunk_length (int, optional): Size in bytes of the file chunks to read at once. Ignored if `json_fd` is None. Do not use! Use `from_file` or `load` instead. Defaults to 1MB.
-
+        stream_stable (bool, optional): When the json to be repaired is the accumulation of streaming json at a certain moment.If this parameter to True will keep the repair results stable.
     Returns:
         Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON or a tuple with the repaired JSON and repair log.
     """
-    parser = JSONParser(json_str, json_fd, logging, chunk_length)
+    parser = JSONParser(json_str, json_fd, logging, chunk_length, stream_stable)
     if skip_json_loads:
         parsed_json = parser.parse()
     else:
@@ -76,6 +77,7 @@ def loads(
     json_str: str,
     skip_json_loads: bool = False,
     logging: bool = False,
+    stream_stable: bool = False,
 ) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     This function works like `json.loads()` except that it will fix your JSON in the process.
@@ -94,6 +96,7 @@ def loads(
         return_objects=True,
         skip_json_loads=skip_json_loads,
         logging=logging,
+        stream_stable=stream_stable,
     )
 
 

{json_repair-0.42.0.dist-info → json_repair-0.44.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: json_repair
-Version: 0.42.0
+Version: 0.44.0
 Summary: A package to repair broken json strings
 Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
 License: MIT License
@@ -188,6 +188,14 @@ Some rules of thumb to use:
 - `skip_json_loads` is faster only if you 100% know that the string is not a valid JSON
 - If you are having issues with escaping pass the string as **raw** string like: `r"string with escaping\""`
 
+### Use json_repair with streaming
+
+Sometimes you are streaming some data and want to repair the JSON coming from it. Normally this won't work but you can pass `stream_stable` to `repair_json()` or `loads()` to make it work:
+
+```
+stream_output = repair_json(stream_input, stream_stable=True)
+```
+
 ### Use json_repair from CLI
 
 Install the library for command-line with:

{json_repair-0.42.0.dist-info → json_repair-0.44.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
 json_repair/__init__.py,sha256=c4L2kZrHvWEKfj_ODU2naliNuvU6FlFVxtF0hbLe6s8,178
 json_repair/__main__.py,sha256=EsJb-y89uZEvGQQg1GdIDWzfDwfOMvVekKEtdguQXCM,67
 json_repair/json_context.py,sha256=mm6dOyrPJ1sDskTORZSXCW7W9-5veMlUKqXQ3Hw3EG4,971
-json_repair/json_parser.py,sha256=bu8FBxaazJ_tRJQmdZA7Me68HD7t3JuReVgjvyGWbmQ,39174
-json_repair/json_repair.py,sha256=k-5HRRlCqrxNmJi0u1KE3IUeL4HXqi1XZ7oAL-NFDLo,10314
+json_repair/json_parser.py,sha256=ID60F0RMzaCpeHPkZbuidJcsmrVBiPmQDRUOgjoeedE,41972
+json_repair/json_repair.py,sha256=o84um759Alft7mlj7lXZFtPQZQPjbo5Jxraa7dTdiRg,10621
 json_repair/object_comparer.py,sha256=SeicB6_N4BHAEPon7s2BELEaJc4oyR9ZhfX2RgPk6Bw,1682
 json_repair/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 json_repair/string_file_wrapper.py,sha256=koZmdq2-Z5K7XF1bDqX6dEbNaVMJYcMTjq-aGe6NQvA,4526
-json_repair-0.42.0.dist-info/licenses/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
-json_repair-0.42.0.dist-info/METADATA,sha256=XJXQxqg6znbpuxK1hTY_0v3FB4KtH_8gOMPD-u_EdvY,11860
-json_repair-0.42.0.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-json_repair-0.42.0.dist-info/entry_points.txt,sha256=SNfge3zPSP-ASqriYU9r3NAPaXdseYr7ciPMKdV2uSw,57
-json_repair-0.42.0.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
-json_repair-0.42.0.dist-info/RECORD,,
+json_repair-0.44.0.dist-info/licenses/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
+json_repair-0.44.0.dist-info/METADATA,sha256=mu_r9oiyo_35hwk745ZTFoMZrJ9PBjRPjFKgICkKSZQ,12157
+json_repair-0.44.0.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
+json_repair-0.44.0.dist-info/entry_points.txt,sha256=SNfge3zPSP-ASqriYU9r3NAPaXdseYr7ciPMKdV2uSw,57
+json_repair-0.44.0.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
+json_repair-0.44.0.dist-info/RECORD,,

{json_repair-0.42.0.dist-info → json_repair-0.44.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.0)
+Generator: setuptools (80.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any