json-repair 0.13.1__py3-none-any.whl → 0.15.0__py3-none-any.whl

json_repair/json_repair.py

@@ -27,17 +27,25 @@ from typing import Any, Dict, List, Union, TextIO
 
 
 class JSONParser:
-    def __init__(self, json_str: str) -> None:
+    def __init__(self, json_str: str, logging: bool = False) -> None:
         # The string to parse
         self.json_str = json_str
         # Index is our iterator that will keep track of which character we are looking at right now
         self.index = 0
         # This is used in the object member parsing to manage the special cases of missing quotes in key or value
-        self.context = ""
-        self.context_stack = []
+        self.context = []
+        # Use this to log the activity, but only if logging is active
+        self.logger = {
+            "log": [],
+            "window": 10,
+            "log_level": "info" if logging else "none",
+        }
 
     def parse(self) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
-        return self.parse_json()
+        if self.logger["log_level"] == "none":
+            return self.parse_json()
+        else:
+            return self.parse_json(), self.logger["log"]
 
     def parse_json(
         self,
@@ -56,7 +64,11 @@ class JSONParser:
             return self.parse_array()
         # there can be an edge case in which a key is empty and at the end of an object
         # like "key": }. We return an empty string here to close the object properly
-        elif char == "}" and self.context == "object_value":
+        elif char == "}" and self.get_context() == "object_value":
+            self.log(
+                "At the end of an object we found a key with missing value, skipping",
+                "info",
+            )
             return ""
         # <string> starts with '"'
         elif char == '"':
@@ -92,13 +104,17 @@ class JSONParser:
 
             # Sometimes LLMs do weird things, if we find a ":" so early, we'll change it to "," and move on
             if (self.get_char_at() or "") == ":":
+                self.log(
+                    "While parsing an object we found a : before a key, replacing with ,",
+                    "info",
+                )
                 self.remove_char_at()
                 self.insert_char_at(",")
                 self.index += 1
 
             # We are now searching for they string key
             # Context is used in the string parser to manage the lack of quotes
-            self.update_context("object_key")
+            self.set_context("object_key")
 
             self.skip_whitespaces_at()
 
@@ -110,6 +126,10 @@ class JSONParser:
                 # This can happen sometimes like { "": "value" }
                 if key == "" and self.get_char_at() == ":":
                     key = "empty_placeholder"
+                    self.log(
+                        "While parsing an object we found an empty key, replacing with empty_placeholder",
+                        "info",
+                    )
                     break
 
             # We reached the end here
@@ -118,15 +138,19 @@ class JSONParser:
 
             # An extreme case of missing ":" after a key
             if (self.get_char_at() or "") != ":":
+                self.log(
+                    "While parsing an object we missed a : after a key, adding it back",
+                    "info",
+                )
                 self.insert_char_at(":")
             self.index += 1
-            self.update_context("")
-            self.update_context("object_value")
+            self.reset_context()
+            self.set_context("object_value")
             # The value can be any valid json
             value = self.parse_json()
 
             # Reset context since our job is done
-            self.update_context("")
+            self.reset_context()
             obj[key] = value
 
             if (self.get_char_at() or "") in [",", "'", '"']:
@@ -137,6 +161,10 @@ class JSONParser:
 
         # Especially at the end of an LLM generated json you might miss the last "}"
         if (self.get_char_at() or "}") != "}":
+            self.log(
+                "While parsing an object, we couldn't find the closing }, adding it back",
+                "info",
+            )
             self.insert_char_at("}")
         self.index += 1
         return obj
@@ -160,15 +188,22 @@ class JSONParser:
                 self.index += 1
                 char = self.get_char_at()
             # If this is the right value of an object and we are closing the object, it means the array is over
-            if self.context == "object_value" and char == "}":
+            if self.get_context() == "object_value" and char == "}":
                 break
 
         # Especially at the end of an LLM generated json you might miss the last "]"
         char = self.get_char_at()
         if char and char != "]":
+            self.log(
+                "While parsing an array we missed the closing ], adding it back", "info"
+            )
             # Sometimes when you fix a missing "]" you'll have a trailing "," there that makes the JSON invalid
             if char == ",":
                 # Remove trailing "," before adding the "]"
+                self.log(
+                    "While parsing an array, remove a trailing , before adding ]",
+                    "info",
+                )
                 self.remove_char_at()
             self.insert_char_at("]")
             self.index -= 1
@@ -183,18 +218,23 @@ class JSONParser:
 
         # Flag to manage corner cases related to missing starting quote
         fixed_quotes = False
-        double_delimiter = False
         lstring_delimiter = rstring_delimiter = '"'
         if isinstance(string_quotes, list):
             lstring_delimiter = string_quotes[0]
             rstring_delimiter = string_quotes[1]
         elif isinstance(string_quotes, str):
             lstring_delimiter = rstring_delimiter = string_quotes
+        # There is sometimes a weird case of doubled quotes, we manage this also later in the while loop
         if self.get_char_at(1) == lstring_delimiter:
-            double_delimiter = True
+            self.log(
+                "While parsing a string, we found a doubled quote, ignoring it", "info"
+            )
             self.index += 1
         char = self.get_char_at()
         if char != lstring_delimiter:
+            self.log(
+                "While parsing a string, we found no starting quote, adding it", "info"
+            )
             self.insert_char_at(lstring_delimiter)
             fixed_quotes = True
         else:
@@ -210,12 +250,13 @@ class JSONParser:
         # * It iterated over the entire sequence
         # * If we are fixing missing quotes in an object, when it finds the special terminators
         char = self.get_char_at()
-        fix_broken_markdown_link = False
         while char and char != rstring_delimiter:
             if fixed_quotes:
-                if self.context == "object_key" and (char == ":" or char.isspace()):
+                if self.get_context() == "object_key" and (
+                    char == ":" or char.isspace()
+                ):
                     break
-                elif self.context == "object_value" and char in [",", "}"]:
+                elif self.get_context() == "object_value" and char in [",", "}"]:
                     break
             self.index += 1
             char = self.get_char_at()
@@ -227,21 +268,46 @@ class JSONParser:
                 else:
                     self.remove_char_at(-1)
                     self.index -= 1
-            # ChatGPT sometimes forget to quote links in markdown like: { "content": "[LINK]("https://google.com")" }
+            # ChatGPT sometimes forget to quote stuff in html tags or markdown, so we do this whole thing here
             if (
                 char == rstring_delimiter
-                # Next character is not a comma
-                and self.get_char_at(1) != ","
-                and (
-                    fix_broken_markdown_link
-                    or (self.get_char_at(-2) == "]" and self.get_char_at(-1)) == "("
-                )
+                # Next character is not a delimiter
+                and self.get_char_at(1) not in [",", ":", "]", "}"]
             ):
-                fix_broken_markdown_link = not fix_broken_markdown_link
-                self.index += 1
-                char = self.get_char_at()
-
-        if char and fixed_quotes and self.context == "object_key" and char.isspace():
+                # Special case here, in case of double quotes one after another
+                if self.get_char_at(1) == rstring_delimiter:
+                    self.log(
+                        "While parsing a string, we found a doubled quote, ignoring it",
+                        "info",
+                    )
+                    # self destruct this character
+                    self.remove_char_at()
+                else:
+                    # Check if eventually there is a rstring delimiter, otherwise we bail
+                    i = 2
+                    next_c = self.get_char_at(i)
+                    while next_c and next_c != rstring_delimiter:
+                        i += 1
+                        next_c = self.get_char_at(i)
+                    # In that case we ignore this rstring delimiter
+                    if next_c:
+                        self.log(
+                            "While parsing a string, we a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
+                            "info",
+                        )
+                        self.index += 1
+                        char = self.get_char_at()
+
+        if (
+            char
+            and fixed_quotes
+            and self.get_context() == "object_key"
+            and char.isspace()
+        ):
+            self.log(
+                "While parsing a string, handling an extreme corner case in which the LLM added a comment instead of valid string, invalidate the string and return an empty value",
+                "info",
+            )
             self.skip_whitespaces_at()
             if self.get_char_at() not in [":", ","]:
                 return ""
@@ -250,13 +316,15 @@ class JSONParser:
 
         # A fallout of the previous special case in the while loop, we need to update the index only if we had a closing quote
         if char != rstring_delimiter:
+            self.log(
+                "While parsing a string, we missed the closing quote, adding it back",
+                "info",
+            )
             self.insert_char_at(rstring_delimiter)
         else:
             self.index += 1
-            if double_delimiter and self.get_char_at() == rstring_delimiter:
-                self.index += 1
 
-        return self.json_str[start:end]
+        return self.json_str[start:end].rstrip()
 
     def parse_number(self) -> Union[float, int, str]:
         # <number> is a valid real number expressed in one of a number of given formats
@@ -325,30 +393,52 @@ class JSONParser:
             except IndexError:
                 return
 
-    def update_context(self, value: str) -> None:
+    def set_context(self, value: str) -> None:
         # If a value is provided update the context variable and save in stack
         if value:
-            if self.context:
-                self.context_stack.append(self.context)
-            self.context = value
-        # Otherwise pop and update the context, or empty if the stack is empty
-        else:
-            try:
-                self.context = self.context_stack.pop()
-            except Exception:
-                self.context = ""
+            self.context.append(value)
+
+    def reset_context(self) -> None:
+        try:
+            self.context.pop()
+        except Exception:
+            return
+
+    def get_context(self) -> str:
+        try:
+            return self.context[0]
+        except Exception:
+            return ""
+
+    def log(self, text: str, level: str) -> None:
+        if level == self.logger["log_level"]:
+            self.logger["log"].append(
+                {
+                    "text": text,
+                    "context": self.json_str[
+                        self.index
+                        - self.logger["window"] : self.index
+                        + self.logger["window"]
+                    ],
+                }
+            )
 
 
 def repair_json(
-    json_str: str, return_objects: bool = False, skip_json_loads: bool = False
+    json_str: str,
+    return_objects: bool = False,
+    skip_json_loads: bool = False,
+    logging: bool = False,
 ) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
     """
     Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
     It will return the fixed string by default.
     When `return_objects=True` is passed, it will return the decoded data structure instead.
+    When `skip_json_loads=True` is passed, it will not call the built-in json.loads() function
+    When `logging=True` is passed, it will return an tuple with the repaired json and a log of all repair actions
     """
     json_str = json_str.strip().lstrip("```json")
-    parser = JSONParser(json_str)
+    parser = JSONParser(json_str, logging)
     if skip_json_loads:
         parsed_json = parser.parse()
     else:
@@ -357,7 +447,7 @@ def repair_json(
         except json.JSONDecodeError:
             parsed_json = parser.parse()
     # It's useful to return the actual object instead of the json string, it allows this lib to be a replacement of the json library
-    if return_objects:
+    if return_objects or logging:
         return parsed_json
     return json.dumps(parsed_json)
 
@@ -384,3 +474,6 @@ def from_file(
     fd.close()
 
     return jsonobj
+
+
+print(repair_json('{ "key": "value", "key2": }', logging=True))

{json_repair-0.13.1.dist-info → json_repair-0.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: json_repair
-Version: 0.13.1
+Version: 0.15.0
 Summary: A package to repair broken json strings
 Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
 License: MIT License
@@ -150,7 +150,10 @@ You will need owner access to this repository
 - Run `python -m build`
 - Create a new release in Github, making sure to tag all the issues solved and contributors. Create the new tag, same as the one in the build configuration
 - Once the release is created, a new Github Actions workflow will start to publish on Pypi, make sure it didn't fail
-
+---
+# Repair JSON in other programming languages
+- Typescript: https://github.com/josdejong/jsonrepair
+- Go: https://github.com/RealAlexandreAI/json-repair
 ---
 # Bonus Content
 If you need some good Custom Instructions (System Message) to improve your chatbot responses try https://gist.github.com/mangiucugna/7ec015c4266df11be8aa510be0110fe4

json_repair-0.15.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+json_repair/__init__.py,sha256=AlNie5y6BZBioGi5fzTAUvum_y0U5aL5aNsuQ_68LQc,175
+json_repair/json_repair.py,sha256=ctuP4AaBrsWBzhF2Al-gX_itHcTG15cqU4Z56KYxNfA,19119
+json_repair-0.15.0.dist-info/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
+json_repair-0.15.0.dist-info/METADATA,sha256=bnhSr8AectNHH-ljyaqwIC5BqPkVf2uYwLHuNuUYpyQ,7355
+json_repair-0.15.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+json_repair-0.15.0.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
+json_repair-0.15.0.dist-info/RECORD,,

json_repair-0.13.1.dist-info/RECORD

@@ -1,7 +0,0 @@
-json_repair/__init__.py,sha256=AlNie5y6BZBioGi5fzTAUvum_y0U5aL5aNsuQ_68LQc,175
-json_repair/json_repair.py,sha256=DB220fZ1BCf--9CeP6AzL2FCk9tpE1Eh3WxgFo33P88,15460
-json_repair-0.13.1.dist-info/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
-json_repair-0.13.1.dist-info/METADATA,sha256=gcfdAU5nhvlHlE3wMvGcwHRQUzgPT2p08cgDg7iyLvw,7200
-json_repair-0.13.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-json_repair-0.13.1.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
-json_repair-0.13.1.dist-info/RECORD,,