json-repair 0.47.4__py3-none-any.whl → 0.47.6__py3-none-any.whl

json_repair/parse_string.py

@@ -0,0 +1,413 @@
+from .constants import STRING_DELIMITERS
+from .json_context import ContextValues
+
+
+def parse_string(self) -> str | bool | None:
+    # <string> is a string of valid characters enclosed in quotes
+    # i.e. { name: "John" }
+    # Somehow all weird cases in an invalid JSON happen to be resolved in this function, so be careful here
+
+    # Flag to manage corner cases related to missing starting quote
+    missing_quotes = False
+    doubled_quotes = False
+    lstring_delimiter = rstring_delimiter = '"'
+
+    char = self.get_char_at()
+    if char in ["#", "/"]:
+        return self.parse_comment()
+    # A valid string can only start with a valid quote or, in our case, with a literal
+    while char and char not in STRING_DELIMITERS and not char.isalnum():
+        self.index += 1
+        char = self.get_char_at()
+
+    if not char:
+        # This is an empty string
+        return ""
+
+    # Ensuring we use the right delimiter
+    if char == "'":
+        lstring_delimiter = rstring_delimiter = "'"
+    elif char == "“":
+        lstring_delimiter = "“"
+        rstring_delimiter = "”"
+    elif char.isalnum():
+        # This could be a <boolean> and not a string. Because (T)rue or (F)alse or (N)ull are valid
+        # But remember, object keys are only of type string
+        if char.lower() in ["t", "f", "n"] and self.context.current != ContextValues.OBJECT_KEY:
+            value = self.parse_boolean_or_null()
+            if value != "":
+                return value
+        self.log(
+            "While parsing a string, we found a literal instead of a quote",
+        )
+        missing_quotes = True
+
+    if not missing_quotes:
+        self.index += 1
+
+    # There is sometimes a weird case of doubled quotes, we manage this also later in the while loop
+    if self.get_char_at() in STRING_DELIMITERS and self.get_char_at() == lstring_delimiter:
+        # If it's an empty key, this was easy
+        if (self.context.current == ContextValues.OBJECT_KEY and self.get_char_at(1) == ":") or (
+            self.context.current == ContextValues.OBJECT_VALUE and self.get_char_at(1) in [",", "}"]
+        ):
+            self.index += 1
+            return ""
+        elif self.get_char_at(1) == lstring_delimiter:
+            # There's something fishy about this, we found doubled quotes and then again quotes
+            self.log(
+                "While parsing a string, we found a doubled quote and then a quote again, ignoring it",
+            )
+            return ""
+        # Find the next delimiter
+        i = self.skip_to_character(character=rstring_delimiter, idx=1)
+        next_c = self.get_char_at(i)
+        # Now check that the next character is also a delimiter to ensure that we have "".....""
+        # In that case we ignore this rstring delimiter
+        if next_c and (self.get_char_at(i + 1) or "") == rstring_delimiter:
+            self.log(
+                "While parsing a string, we found a valid starting doubled quote",
+            )
+            doubled_quotes = True
+            self.index += 1
+        else:
+            # Ok this is not a doubled quote, check if this is an empty string or not
+            i = self.skip_whitespaces_at(idx=1, move_main_index=False)
+            next_c = self.get_char_at(i)
+            if next_c in STRING_DELIMITERS + ["{", "["]:
+                # something fishy is going on here
+                self.log(
+                    "While parsing a string, we found a doubled quote but also another quote afterwards, ignoring it",
+                )
+                self.index += 1
+                return ""
+            elif next_c not in [",", "]", "}"]:
+                self.log(
+                    "While parsing a string, we found a doubled quote but it was a mistake, removing one quote",
+                )
+                self.index += 1
+
+    # Initialize our return value
+    string_acc = ""
+
+    # Here things get a bit hairy because a string missing the final quote can also be a key or a value in an object
+    # In that case we need to use the ":|,|}" characters as terminators of the string
+    # So this will stop if:
+    # * It finds a closing quote
+    # * It iterated over the entire sequence
+    # * If we are fixing missing quotes in an object, when it finds the special terminators
+    char = self.get_char_at()
+    unmatched_delimiter = False
+    while char and char != rstring_delimiter:
+        if missing_quotes and self.context.current == ContextValues.OBJECT_KEY and (char == ":" or char.isspace()):
+            self.log(
+                "While parsing a string missing the left delimiter in object key context, we found a :, stopping here",
+            )
+            break
+        if (
+            not self.stream_stable
+            and self.context.current == ContextValues.OBJECT_VALUE
+            and char
+            in [
+                ",",
+                "}",
+            ]
+            and (not string_acc or string_acc[-1] != rstring_delimiter)
+        ):
+            rstring_delimiter_missing = True
+            # check if this is a case in which the closing comma is NOT missing instead
+            self.skip_whitespaces_at()
+            if self.get_char_at(1) == "\\":
+                # Ok this is a quoted string, skip
+                rstring_delimiter_missing = False
+            i = self.skip_to_character(character=rstring_delimiter, idx=1)
+            next_c = self.get_char_at(i)
+            if next_c:
+                i += 1
+                # found a delimiter, now we need to check that is followed strictly by a comma or brace
+                # or the string ended
+                i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                next_c = self.get_char_at(i)
+                if not next_c or next_c in [",", "}"]:
+                    rstring_delimiter_missing = False
+                else:
+                    # OK but this could still be some garbage at the end of the string
+                    # So we need to check if we find a new lstring_delimiter afterwards
+                    # If we do, maybe this is a missing delimiter
+                    i = self.skip_to_character(character=lstring_delimiter, idx=i)
+                    next_c = self.get_char_at(i)
+                    if not next_c:
+                        rstring_delimiter_missing = False
+                    else:
+                        # But again, this could just be something a bit stupid like "lorem, "ipsum" sic"
+                        # Check if we find a : afterwards (skipping space)
+                        i = self.skip_whitespaces_at(idx=i + 1, move_main_index=False)
+                        next_c = self.get_char_at(i)
+                        if next_c and next_c != ":":
+                            rstring_delimiter_missing = False
+            else:
+                # There could be a case in which even the next key:value is missing delimeters
+                # because it might be a systemic issue with the output
+                # So let's check if we can find a : in the string instead
+                i = self.skip_to_character(character=":", idx=1)
+                next_c = self.get_char_at(i)
+                if next_c:
+                    # OK then this is a systemic issue with the output
+                    break
+                else:
+                    # skip any whitespace first
+                    i = self.skip_whitespaces_at(idx=1, move_main_index=False)
+                    # We couldn't find any rstring_delimeter before the end of the string
+                    # check if this is the last string of an object and therefore we can keep going
+                    # make an exception if this is the last char before the closing brace
+                    j = self.skip_to_character(character="}", idx=i)
+                    if j - i > 1:
+                        # Ok it's not right after the comma
+                        # Let's ignore
+                        rstring_delimiter_missing = False
+                    # Check that j was not out of bound
+                    elif self.get_char_at(j):
+                        # Check for an unmatched opening brace in string_acc
+                        for c in reversed(string_acc):
+                            if c == "{":
+                                # Ok then this is part of the string
+                                rstring_delimiter_missing = False
+                                break
+            if rstring_delimiter_missing:
+                self.log(
+                    "While parsing a string missing the left delimiter in object value context, we found a , or } and we couldn't determine that a right delimiter was present. Stopping here",
+                )
+                break
+        if (
+            not self.stream_stable
+            and char == "]"
+            and ContextValues.ARRAY in self.context.context
+            and string_acc[-1] != rstring_delimiter
+        ):
+            # We found the end of an array and we are in array context
+            # So let's check if we find a rstring_delimiter forward otherwise end early
+            i = self.skip_to_character(rstring_delimiter)
+            if not self.get_char_at(i):
+                # No delimiter found
+                break
+        string_acc += char
+        self.index += 1
+        char = self.get_char_at()
+        # Unclosed string ends with a \ character. This character is ignored if stream_stable = True.
+        if self.stream_stable and not char and string_acc[-1] == "\\":
+            string_acc = string_acc[:-1]
+        if char and string_acc[-1] == "\\":
+            # This is a special case, if people use real strings this might happen
+            self.log("Found a stray escape sequence, normalizing it")
+            if char in [rstring_delimiter, "t", "n", "r", "b", "\\"]:
+                string_acc = string_acc[:-1]
+                escape_seqs = {"t": "\t", "n": "\n", "r": "\r", "b": "\b"}
+                string_acc += escape_seqs.get(char, char)
+                self.index += 1
+                char = self.get_char_at()
+                while char and string_acc[-1] == "\\" and char in [rstring_delimiter, "\\"]:
+                    # this is a bit of a special case, if I don't do this it will close the loop or create a train of \\
+                    # I don't love it though
+                    string_acc = string_acc[:-1]
+                    string_acc += char
+                    self.index += 1
+                    char = self.get_char_at()
+                continue
+            elif char in ["u", "x"]:
+                # If we find a unicode escape sequence, normalize it
+                num_chars = 4 if char == "u" else 2
+                next_chars = self.json_str[self.index + 1 : self.index + 1 + num_chars]
+                if len(next_chars) == num_chars and all(c in "0123456789abcdefABCDEF" for c in next_chars):
+                    self.log("Found a unicode escape sequence, normalizing it")
+                    string_acc = string_acc[:-1]
+                    string_acc += chr(int(next_chars, 16))
+                    self.index += 1 + num_chars
+                    char = self.get_char_at()
+                    continue
+        # If we are in object key context and we find a colon, it could be a missing right quote
+        if char == ":" and not missing_quotes and self.context.current == ContextValues.OBJECT_KEY:
+            # Ok now we need to check if this is followed by a value like "..."
+            i = self.skip_to_character(character=lstring_delimiter, idx=1)
+            next_c = self.get_char_at(i)
+            if next_c:
+                i += 1
+                # found the first delimiter
+                i = self.skip_to_character(character=rstring_delimiter, idx=i)
+                next_c = self.get_char_at(i)
+                if next_c:
+                    # found a second delimiter
+                    i += 1
+                    # Skip spaces
+                    i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                    next_c = self.get_char_at(i)
+                    if next_c and next_c in [",", "}"]:
+                        # Ok then this is a missing right quote
+                        self.log(
+                            "While parsing a string missing the right delimiter in object key context, we found a :, stopping here",
+                        )
+                        break
+            else:
+                # The string ended without finding a lstring_delimiter, I will assume this is a missing right quote
+                self.log(
+                    "While parsing a string missing the right delimiter in object key context, we found a :, stopping here",
+                )
+                break
+        # ChatGPT sometimes forget to quote stuff in html tags or markdown, so we do this whole thing here
+        if char == rstring_delimiter and string_acc[-1] != "\\":
+            # Special case here, in case of double quotes one after another
+            if doubled_quotes and self.get_char_at(1) == rstring_delimiter:
+                self.log("While parsing a string, we found a doubled quote, ignoring it")
+                self.index += 1
+            elif missing_quotes and self.context.current == ContextValues.OBJECT_VALUE:
+                # In case of missing starting quote I need to check if the delimeter is the end or the beginning of a key
+                i = 1
+                next_c = self.get_char_at(i)
+                while next_c and next_c not in [
+                    rstring_delimiter,
+                    lstring_delimiter,
+                ]:
+                    i += 1
+                    next_c = self.get_char_at(i)
+                if next_c:
+                    # We found a quote, now let's make sure there's a ":" following
+                    i += 1
+                    # found a delimiter, now we need to check that is followed strictly by a comma or brace
+                    i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                    next_c = self.get_char_at(i)
+                    if next_c and next_c == ":":
+                        # Reset the cursor
+                        self.index -= 1
+                        char = self.get_char_at()
+                        self.log(
+                            "In a string with missing quotes and object value context, I found a delimeter but it turns out it was the beginning on the next key. Stopping here.",
+                        )
+                        break
+            elif unmatched_delimiter:
+                unmatched_delimiter = False
+                string_acc += str(char)
+                self.index += 1
+                char = self.get_char_at()
+            else:
+                # Check if eventually there is a rstring delimiter, otherwise we bail
+                i = 1
+                next_c = self.get_char_at(i)
+                check_comma_in_object_value = True
+                while next_c and next_c not in [
+                    rstring_delimiter,
+                    lstring_delimiter,
+                ]:
+                    # This is a bit of a weird workaround, essentially in object_value context we don't always break on commas
+                    # This is because the routine after will make sure to correct any bad guess and this solves a corner case
+                    if check_comma_in_object_value and next_c.isalpha():
+                        check_comma_in_object_value = False
+                    # If we are in an object context, let's check for the right delimiters
+                    if (
+                        (ContextValues.OBJECT_KEY in self.context.context and next_c in [":", "}"])
+                        or (ContextValues.OBJECT_VALUE in self.context.context and next_c == "}")
+                        or (ContextValues.ARRAY in self.context.context and next_c in ["]", ","])
+                        or (
+                            check_comma_in_object_value
+                            and self.context.current == ContextValues.OBJECT_VALUE
+                            and next_c == ","
+                        )
+                    ):
+                        break
+                    i += 1
+                    next_c = self.get_char_at(i)
+                # If we stopped for a comma in object_value context, let's check if find a "} at the end of the string
+                if next_c == "," and self.context.current == ContextValues.OBJECT_VALUE:
+                    i += 1
+                    i = self.skip_to_character(character=rstring_delimiter, idx=i)
+                    next_c = self.get_char_at(i)
+                    # Ok now I found a delimiter, let's skip whitespaces and see if next we find a }
+                    i += 1
+                    i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                    next_c = self.get_char_at(i)
+                elif next_c == rstring_delimiter and self.get_char_at(i - 1) != "\\":
+                    # Check if self.index:self.index+i is only whitespaces, break if that's the case
+                    if all(str(self.get_char_at(j)).isspace() for j in range(1, i) if self.get_char_at(j)):
+                        break
+                    if self.context.current == ContextValues.OBJECT_VALUE:
+                        i = self.skip_whitespaces_at(idx=i + 1, move_main_index=False)
+                        if self.get_char_at(i) == ",":
+                            # So we found a comma, this could be a case of a single quote like "va"lue",
+                            # Search if it's followed by another key, starting with the first delimeter
+                            i = self.skip_to_character(character=lstring_delimiter, idx=i + 1)
+                            i += 1
+                            i = self.skip_to_character(character=rstring_delimiter, idx=i + 1)
+                            i += 1
+                            i = self.skip_whitespaces_at(idx=i, move_main_index=False)
+                            next_c = self.get_char_at(i)
+                            if next_c == ":":
+                                self.log(
+                                    "While parsing a string, we a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
+                                )
+                                string_acc += str(char)
+                                self.index += 1
+                                char = self.get_char_at()
+                                continue
+                        # We found a delimiter and we need to check if this is a key
+                        # so find a rstring_delimiter and a colon after
+                        i = self.skip_to_character(character=rstring_delimiter, idx=i + 1)
+                        i += 1
+                        next_c = self.get_char_at(i)
+                        while next_c and next_c != ":":
+                            if next_c in [",", "]", "}"] or (
+                                next_c == rstring_delimiter and self.get_char_at(i - 1) != "\\"
+                            ):
+                                break
+                            i += 1
+                            next_c = self.get_char_at(i)
+                        # Only if we fail to find a ':' then we know this is misplaced quote
+                        if next_c != ":":
+                            self.log(
+                                "While parsing a string, we a misplaced quote that would have closed the string but has a different meaning here, ignoring it",
+                            )
+                            unmatched_delimiter = not unmatched_delimiter
+                            string_acc += str(char)
+                            self.index += 1
+                            char = self.get_char_at()
+                    elif self.context.current == ContextValues.ARRAY:
+                        # If we got up to here it means that this is a situation like this:
+                        # ["bla bla bla "puppy" bla bla bla "kitty" bla bla"]
+                        # So we need to ignore this quote
+                        self.log(
+                            "While parsing a string in Array context, we detected a quoted section that would have closed the string but has a different meaning here, ignoring it",
+                        )
+                        unmatched_delimiter = not unmatched_delimiter
+                        string_acc += str(char)
+                        self.index += 1
+                        char = self.get_char_at()
+                    elif self.context.current == ContextValues.OBJECT_KEY:
+                        # In this case we just ignore this and move on
+                        self.log(
+                            "While parsing a string in Object Key context, we detected a quoted section that would have closed the string but has a different meaning here, ignoring it",
+                        )
+                        string_acc += str(char)
+                        self.index += 1
+                        char = self.get_char_at()
+    if char and missing_quotes and self.context.current == ContextValues.OBJECT_KEY and char.isspace():
+        self.log(
+            "While parsing a string, handling an extreme corner case in which the LLM added a comment instead of valid string, invalidate the string and return an empty value",
+        )
+        self.skip_whitespaces_at()
+        if self.get_char_at() not in [":", ","]:
+            return ""
+
+    # A fallout of the previous special case in the while loop,
+    # we need to update the index only if we had a closing quote
+    if char != rstring_delimiter:
+        # if stream_stable = True, unclosed strings do not trim trailing whitespace characters
+        if not self.stream_stable:
+            self.log(
+                "While parsing a string, we missed the closing quote, ignoring",
+            )
+            string_acc = string_acc.rstrip()
+    else:
+        self.index += 1
+
+    if not self.stream_stable and (missing_quotes or (string_acc and string_acc[-1] == "\n")):
+        # Clean the whitespaces for some corner cases
+        string_acc = string_acc.rstrip()
+
+    return string_acc

{json_repair-0.47.4.dist-info → json_repair-0.47.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: json_repair
-Version: 0.47.4
+Version: 0.47.6
 Summary: A package to repair broken json strings
 Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
 License: MIT License

json_repair-0.47.6.dist-info/RECORD

@@ -0,0 +1,21 @@
+json_repair/__init__.py,sha256=JdJIZNCKV3MfIviryqK8NH8yGssCta2-192CekcwH-o,174
+json_repair/__main__.py,sha256=EsJb-y89uZEvGQQg1GdIDWzfDwfOMvVekKEtdguQXCM,67
+json_repair/constants.py,sha256=cv2gvyosuq0me0600WyTysM9avrtfXPuXYR26tawcuo,158
+json_repair/json_context.py,sha256=WsMOjqpGSr6aaDONcrk8UFtTurzWon2Qq9AoBBYseoI,934
+json_repair/json_parser.py,sha256=WpU8K3E51gJ9BKbuW7LcMQGXWArte8noYMzvA9qu6Wc,6850
+json_repair/json_repair.py,sha256=txblCJtcTpXcQaT15tavulkJPtyRYe2cfYpPHZcvPv0,11233
+json_repair/object_comparer.py,sha256=LlIF0MisRglzC-CiG5AxAEDCBWBHeJd-6uXYx0uRmCk,1175
+json_repair/parse_array.py,sha256=YnESGXnRaX57zXv7NP6EcHOlqgeaLEzOy1s_l9ghTeY,2002
+json_repair/parse_boolean_or_null.py,sha256=2KoUkjiZ68fkge_n_Q4bbFVG6WskRroKD55jW2Ep2OU,782
+json_repair/parse_comment.py,sha256=kNTinpdHZftrtV190-Julq5eKFxMSmGYNbwlj4vGtsg,2492
+json_repair/parse_number.py,sha256=o7wEER7_H6xG0WsmvKS8VucoMJ7AsaJdxkDzulJ9o-Q,1192
+json_repair/parse_object.py,sha256=yQ9SilLdBBW5cYJOcQGB4ZR8MtOp4cH6WilfXF_kdgE,4456
+json_repair/parse_string.py,sha256=8f-zr9I2n5tAHkot0pBzqcn8avZbhEb-CX0NLwQUMUU,22131
+json_repair/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+json_repair/string_file_wrapper.py,sha256=tGkWBEUPE-CZPf4uSM5NE9oSDTpskX0myJiXsl-gbds,4333
+json_repair-0.47.6.dist-info/licenses/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
+json_repair-0.47.6.dist-info/METADATA,sha256=S7fZgSot13SzgMXXk4Qh4PJEmDyUJMUy_Vb6FuLW6v8,12411
+json_repair-0.47.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+json_repair-0.47.6.dist-info/entry_points.txt,sha256=SNfge3zPSP-ASqriYU9r3NAPaXdseYr7ciPMKdV2uSw,57
+json_repair-0.47.6.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
+json_repair-0.47.6.dist-info/RECORD,,

json_repair-0.47.4.dist-info/RECORD

@@ -1,14 +0,0 @@
-json_repair/__init__.py,sha256=6FDD6dEVM5Pb5o4Zodgw4ex30Hzy-YvNRy0vts9SQ4I,118
-json_repair/__main__.py,sha256=EsJb-y89uZEvGQQg1GdIDWzfDwfOMvVekKEtdguQXCM,67
-json_repair/json_context.py,sha256=WsMOjqpGSr6aaDONcrk8UFtTurzWon2Qq9AoBBYseoI,934
-json_repair/json_parser.py,sha256=412X7sLnSjN3Rk8l0kSSeNbrDn9wrEgHT31Non-qYzU,40567
-json_repair/json_repair.py,sha256=0qL2LuzlNJa3VnEqYNaJyZNAL2w18oAt2YvA-TlMxmY,11211
-json_repair/object_comparer.py,sha256=LlIF0MisRglzC-CiG5AxAEDCBWBHeJd-6uXYx0uRmCk,1175
-json_repair/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-json_repair/string_file_wrapper.py,sha256=tGkWBEUPE-CZPf4uSM5NE9oSDTpskX0myJiXsl-gbds,4333
-json_repair-0.47.4.dist-info/licenses/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
-json_repair-0.47.4.dist-info/METADATA,sha256=FU5u40Z4wL4VhYHE72CyO3CstJkv1TV1PT-wn_Oracs,12411
-json_repair-0.47.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-json_repair-0.47.4.dist-info/entry_points.txt,sha256=SNfge3zPSP-ASqriYU9r3NAPaXdseYr7ciPMKdV2uSw,57
-json_repair-0.47.4.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
-json_repair-0.47.4.dist-info/RECORD,,