json-repair 0.14.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

json_repair/json_repair.py

@@ -27,17 +27,25 @@ from typing import Any, Dict, List, Union, TextIO
 
 
 class JSONParser:
-    def __init__(self, json_str: str) -> None:
+    def __init__(self, json_str: str, logging: bool = False) -> None:
         # The string to parse
         self.json_str = json_str
         # Index is our iterator that will keep track of which character we are looking at right now
         self.index = 0
         # This is used in the object member parsing to manage the special cases of missing quotes in key or value
-        self.context = ""
-        self.context_stack = []
+        self.context = []
+        # Use this to log the activity, but only if logging is active
+        self.logger = {
+            "log": [],
+            "window": 10,
+            "log_level": "info" if logging else "none",
+        }
 
     def parse(self) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
-        return self.parse_json()
+        if self.logger["log_level"] == "none":
+            return self.parse_json()
+        else:
+            return self.parse_json(), self.logger["log"]
 
     def parse_json(
         self,
@@ -56,7 +64,11 @@ class JSONParser:
             return self.parse_array()
         # there can be an edge case in which a key is empty and at the end of an object
         # like "key": }. We return an empty string here to close the object properly
-        elif char == "}" and self.context == "object_value":
+        elif char == "}" and self.get_context() == "object_value":
+            self.log(
+                "At the end of an object we found a key with missing value, skipping",
+                "info",
+            )
             return ""
         # <string> starts with '"'
         elif char == '"':
@@ -92,13 +104,17 @@ class JSONParser:
 
             # Sometimes LLMs do weird things, if we find a ":" so early, we'll change it to "," and move on
             if (self.get_char_at() or "") == ":":
+                self.log(
+                    "While parsing an object we found a : before a key, replacing with ,",
+                    "info",
+                )
                 self.remove_char_at()
                 self.insert_char_at(",")
                 self.index += 1
 
             # We are now searching for they string key
             # Context is used in the string parser to manage the lack of quotes
-            self.update_context("object_key")
+            self.set_context("object_key")
 
             self.skip_whitespaces_at()
 
@@ -110,6 +126,10 @@ class JSONParser:
                 # This can happen sometimes like { "": "value" }
                 if key == "" and self.get_char_at() == ":":
                     key = "empty_placeholder"
+                    self.log(
+                        "While parsing an object we found an empty key, replacing with empty_placeholder",
+                        "info",
+                    )
                     break
 
             # We reached the end here
@@ -118,15 +138,19 @@ class JSONParser:
 
             # An extreme case of missing ":" after a key
             if (self.get_char_at() or "") != ":":
+                self.log(
+                    "While parsing an object we missed a : after a key, adding it back",
+                    "info",
+                )
                 self.insert_char_at(":")
             self.index += 1
-            self.update_context("")
-            self.update_context("object_value")
+            self.reset_context()
+            self.set_context("object_value")
             # The value can be any valid json
             value = self.parse_json()
 
             # Reset context since our job is done
-            self.update_context("")
+            self.reset_context()
             obj[key] = value
 
             if (self.get_char_at() or "") in [",", "'", '"']:
@@ -137,6 +161,10 @@ class JSONParser:
 
         # Especially at the end of an LLM generated json you might miss the last "}"
         if (self.get_char_at() or "}") != "}":
+            self.log(
+                "While parsing an object, we couldn't find the closing }, adding it back",
+                "info",
+            )
             self.insert_char_at("}")
         self.index += 1
         return obj
@@ -160,15 +188,22 @@ class JSONParser:
                 self.index += 1
                 char = self.get_char_at()
             # If this is the right value of an object and we are closing the object, it means the array is over
-            if self.context == "object_value" and char == "}":
+            if self.get_context() == "object_value" and char == "}":
                 break
 
         # Especially at the end of an LLM generated json you might miss the last "]"
         char = self.get_char_at()
         if char and char != "]":
+            self.log(
+                "While parsing an array we missed the closing ], adding it back", "info"
+            )
             # Sometimes when you fix a missing "]" you'll have a trailing "," there that makes the JSON invalid
             if char == ",":
                 # Remove trailing "," before adding the "]"
+                self.log(
+                    "While parsing an array, remove a trailing , before adding ]",
+                    "info",
+                )
                 self.remove_char_at()
             self.insert_char_at("]")
             self.index -= 1
@@ -191,9 +226,15 @@ class JSONParser:
             lstring_delimiter = rstring_delimiter = string_quotes
         # There is sometimes a weird case of doubled quotes, we manage this also later in the while loop
         if self.get_char_at(1) == lstring_delimiter:
+            self.log(
+                "While parsing a string, we found a doubled quote, ignoring it", "info"
+            )
             self.index += 1
         char = self.get_char_at()
         if char != lstring_delimiter:
+            self.log(
+                "While parsing a string, we found no starting quote, adding it", "info"
+            )
             self.insert_char_at(lstring_delimiter)
             fixed_quotes = True
         else:
@@ -211,9 +252,11 @@ class JSONParser:
         char = self.get_char_at()
         while char and char != rstring_delimiter:
             if fixed_quotes:
-                if self.context == "object_key" and (char == ":" or char.isspace()):
+                if self.get_context() == "object_key" and (
+                    char == ":" or char.isspace()
+                ):
                     break
-                elif self.context == "object_value" and char in [",", "}"]:
+                elif self.get_context() == "object_value" and char in [",", "}"]:
                     break
             self.index += 1
             char = self.get_char_at()
@@ -233,6 +276,10 @@ class JSONParser:
             ):
                 # Special case here, in case of double quotes one after another
                 if self.get_char_at(1) == rstring_delimiter:
+                    self.log(
+                        "While parsing a string, we found a doubled quote, ignoring it",
+                        "info",
+                    )
                     # self destruct this character
                     self.remove_char_at()
                 else:
@@ -244,10 +291,23 @@ class JSONParser:
                         next_c = self.get_char_at(i)
                     # In that case we ignore this rstring delimiter
                     if next_c:
+                        self.log(
+                            "While parsing a string, we a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
+                            "info",
+                        )
                         self.index += 1
                         char = self.get_char_at()
 
-        if char and fixed_quotes and self.context == "object_key" and char.isspace():
+        if (
+            char
+            and fixed_quotes
+            and self.get_context() == "object_key"
+            and char.isspace()
+        ):
+            self.log(
+                "While parsing a string, handling an extreme corner case in which the LLM added a comment instead of valid string, invalidate the string and return an empty value",
+                "info",
+            )
             self.skip_whitespaces_at()
             if self.get_char_at() not in [":", ","]:
                 return ""
@@ -256,6 +316,10 @@ class JSONParser:
 
         # A fallout of the previous special case in the while loop, we need to update the index only if we had a closing quote
         if char != rstring_delimiter:
+            self.log(
+                "While parsing a string, we missed the closing quote, adding it back",
+                "info",
+            )
             self.insert_char_at(rstring_delimiter)
         else:
             self.index += 1
@@ -329,30 +393,52 @@ class JSONParser:
             except IndexError:
                 return
 
-    def update_context(self, value: str) -> None:
+    def set_context(self, value: str) -> None:
         # If a value is provided update the context variable and save in stack
         if value:
-            if self.context:
-                self.context_stack.append(self.context)
-            self.context = value
-        # Otherwise pop and update the context, or empty if the stack is empty
-        else:
-            try:
-                self.context = self.context_stack.pop()
-            except Exception:
-                self.context = ""
+            self.context.append(value)
+
+    def reset_context(self) -> None:
+        try:
+            self.context.pop()
+        except Exception:
+            return
+
+    def get_context(self) -> str:
+        try:
+            return self.context[0]
+        except Exception:
+            return ""
+
+    def log(self, text: str, level: str) -> None:
+        if level == self.logger["log_level"]:
+            self.logger["log"].append(
+                {
+                    "text": text,
+                    "context": self.json_str[
+                        self.index
+                        - self.logger["window"] : self.index
+                        + self.logger["window"]
+                    ],
+                }
+            )
 
 
 def repair_json(
-    json_str: str, return_objects: bool = False, skip_json_loads: bool = False
+    json_str: str,
+    return_objects: bool = False,
+    skip_json_loads: bool = False,
+    logging: bool = False,
 ) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
     """
     Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
     It will return the fixed string by default.
     When `return_objects=True` is passed, it will return the decoded data structure instead.
+    When `skip_json_loads=True` is passed, it will not call the built-in json.loads() function
+    When `logging=True` is passed, it will return an tuple with the repaired json and a log of all repair actions
     """
     json_str = json_str.strip().lstrip("```json")
-    parser = JSONParser(json_str)
+    parser = JSONParser(json_str, logging)
     if skip_json_loads:
         parsed_json = parser.parse()
     else:
@@ -361,30 +447,32 @@ def repair_json(
         except json.JSONDecodeError:
             parsed_json = parser.parse()
     # It's useful to return the actual object instead of the json string, it allows this lib to be a replacement of the json library
-    if return_objects:
+    if return_objects or logging:
         return parsed_json
     return json.dumps(parsed_json)
 
 
 def loads(
-    json_str: str,
+    json_str: str, skip_json_loads: bool = False, logging: bool = False
 ) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
     """
     This function works like `json.loads()` except that it will fix your JSON in the process.
     It is a wrapper around the `repair_json()` function with `return_objects=True`.
     """
-    return repair_json(json_str, True)
+    return repair_json(json_str, True, skip_json_loads, logging)
 
 
-def load(fp: TextIO) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
-    return loads(fp.read())
+def load(
+    fp: TextIO, skip_json_loads: bool = False, logging: bool = False
+) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
+    return loads(fp.read(), skip_json_loads, logging)
 
 
 def from_file(
-    filename: str,
+    filename: str, skip_json_loads: bool = False, logging: bool = False
 ) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
     fd = open(filename)
-    jsonobj = load(fd)
+    jsonobj = load(fd, skip_json_loads, logging)
     fd.close()
 
     return jsonobj

{json_repair-0.14.0.dist-info → json_repair-0.15.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: json_repair
-Version: 0.14.0
+Version: 0.15.1
 Summary: A package to repair broken json strings
 Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
 License: MIT License

json_repair-0.15.1.dist-info/RECORD

@@ -0,0 +1,7 @@
+json_repair/__init__.py,sha256=AlNie5y6BZBioGi5fzTAUvum_y0U5aL5aNsuQ_68LQc,175
+json_repair/json_repair.py,sha256=oGD3DDU_Gni7HA25Mlnx4b5_b7c8F_kRSNrxAR3f9sk,19297
+json_repair-0.15.1.dist-info/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
+json_repair-0.15.1.dist-info/METADATA,sha256=Kpv-aRkvY2N4YOLS9z61lKtD5IeoN0qg74Z0-y7mHtk,7355
+json_repair-0.15.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+json_repair-0.15.1.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
+json_repair-0.15.1.dist-info/RECORD,,

json_repair-0.14.0.dist-info/RECORD

@@ -1,7 +0,0 @@
-json_repair/__init__.py,sha256=AlNie5y6BZBioGi5fzTAUvum_y0U5aL5aNsuQ_68LQc,175
-json_repair/json_repair.py,sha256=8B5HfWoLlUUtRYq1cnbajOxWiMSD9nxNW2cRFPjFfVE,15817
-json_repair-0.14.0.dist-info/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
-json_repair-0.14.0.dist-info/METADATA,sha256=82KldmuVFLXbNy6SXar9MsulkcBUM1K8RX13pNysHQU,7355
-json_repair-0.14.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-json_repair-0.14.0.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
-json_repair-0.14.0.dist-info/RECORD,,