json-repair 0.47.4__py3-none-any.whl → 0.47.6__py3-none-any.whl

json_repair/json_repair.py

@@ -27,7 +27,8 @@ import json
 import sys
 from typing import Literal, TextIO, overload
 
-from .json_parser import JSONParser, JSONReturnType
+from .constants import JSONReturnType
+from .json_parser import JSONParser
 
 
 @overload

json_repair/parse_array.py

@@ -0,0 +1,50 @@
+from .constants import STRING_DELIMITERS, JSONReturnType
+from .json_context import ContextValues
+
+
+def parse_array(self) -> list[JSONReturnType]:
+    # <array> ::= '[' [ <json> *(', ' <json>) ] ']' ; A sequence of JSON values separated by commas
+    arr = []
+    self.context.set(ContextValues.ARRAY)
+    # Stop when you either find the closing parentheses or you have iterated over the entire string
+    char = self.get_char_at()
+    while char and char not in ["]", "}"]:
+        self.skip_whitespaces_at()
+        value: JSONReturnType = ""
+        if char in STRING_DELIMITERS:
+            # Sometimes it can happen that LLMs forget to start an object and then you think it's a string in an array
+            # So we are going to check if this string is followed by a : or not
+            # And either parse the string or parse the object
+            i = 1
+            i = self.skip_to_character(char, i)
+            i = self.skip_whitespaces_at(idx=i + 1, move_main_index=False)
+            value = self.parse_object() if self.get_char_at(i) == ":" else self.parse_string()
+        else:
+            value = self.parse_json()
+
+        # It is possible that parse_json() returns nothing valid, so we increase by 1
+        if value == "":
+            self.index += 1
+        elif value == "..." and self.get_char_at(-1) == ".":
+            self.log(
+                "While parsing an array, found a stray '...'; ignoring it",
+            )
+        else:
+            arr.append(value)
+
+        # skip over whitespace after a value but before closing ]
+        char = self.get_char_at()
+        while char and char != "]" and (char.isspace() or char == ","):
+            self.index += 1
+            char = self.get_char_at()
+
+    # Especially at the end of an LLM generated json you might miss the last "]"
+    if char and char != "]":
+        self.log(
+            "While parsing an array we missed the closing ], ignoring it",
+        )
+
+    self.index += 1
+
+    self.context.reset()
+    return arr

json_repair/parse_boolean_or_null.py

@@ -0,0 +1,24 @@
+def parse_boolean_or_null(self) -> bool | str | None:
+    # <boolean> is one of the literal strings 'true', 'false', or 'null' (unquoted)
+    starting_index = self.index
+    char = (self.get_char_at() or "").lower()
+    value: tuple[str, bool | None] | None = None
+    if char == "t":
+        value = ("true", True)
+    elif char == "f":
+        value = ("false", False)
+    elif char == "n":
+        value = ("null", None)
+
+    if value:
+        i = 0
+        while char and i < len(value[0]) and char == value[0][i]:
+            i += 1
+            self.index += 1
+            char = (self.get_char_at() or "").lower()
+        if i == len(value[0]):
+            return value[1]
+
+    # If nothing works reset the index before returning
+    self.index = starting_index
+    return ""

json_repair/parse_comment.py

@@ -0,0 +1,65 @@
+from .json_context import ContextValues
+
+
+def parse_comment(self) -> str:
+    """
+    Parse code-like comments:
+
+    - "# comment": A line comment that continues until a newline.
+    - "// comment": A line comment that continues until a newline.
+    - "/* comment */": A block comment that continues until the closing delimiter "*/".
+
+    The comment is skipped over and an empty string is returned so that comments do not interfere
+    with the actual JSON elements.
+    """
+    char = self.get_char_at()
+    termination_characters = ["\n", "\r"]
+    if ContextValues.ARRAY in self.context.context:
+        termination_characters.append("]")
+    if ContextValues.OBJECT_VALUE in self.context.context:
+        termination_characters.append("}")
+    if ContextValues.OBJECT_KEY in self.context.context:
+        termination_characters.append(":")
+    # Line comment starting with #
+    if char == "#":
+        comment = ""
+        while char and char not in termination_characters:
+            comment += char
+            self.index += 1
+            char = self.get_char_at()
+        self.log(f"Found line comment: {comment}, ignoring")
+    # Comments starting with '/'
+    elif char == "/":
+        next_char = self.get_char_at(1)
+        # Handle line comment starting with //
+        if next_char == "/":
+            comment = "//"
+            self.index += 2  # Skip both slashes.
+            char = self.get_char_at()
+            while char and char not in termination_characters:
+                comment += char
+                self.index += 1
+                char = self.get_char_at()
+            self.log(f"Found line comment: {comment}, ignoring")
+        # Handle block comment starting with /*
+        elif next_char == "*":
+            comment = "/*"
+            self.index += 2  # Skip '/*'
+            while True:
+                char = self.get_char_at()
+                if not char:
+                    self.log("Reached end-of-string while parsing block comment; unclosed block comment.")
+                    break
+                comment += char
+                self.index += 1
+                if comment.endswith("*/"):
+                    break
+            self.log(f"Found block comment: {comment}, ignoring")
+        else:
+            # Skip standalone '/' characters that are not part of a comment
+            # to avoid getting stuck in an infinite loop
+            self.index += 1
+    if self.context.empty:
+        return self.parse_json()
+    else:
+        return ""

json_repair/parse_number.py

@@ -0,0 +1,32 @@
+from .constants import JSONReturnType
+from .json_context import ContextValues
+
+NUMBER_CHARS: set[str] = set("0123456789-.eE/,")
+
+
+def parse_number(self) -> float | int | str | JSONReturnType:
+    # <number> is a valid real number expressed in one of a number of given formats
+    number_str = ""
+    char = self.get_char_at()
+    is_array = self.context.current == ContextValues.ARRAY
+    while char and char in NUMBER_CHARS and (not is_array or char != ","):
+        number_str += char
+        self.index += 1
+        char = self.get_char_at()
+    if number_str and number_str[-1] in "-eE/,":
+        # The number ends with a non valid character for a number/currency, rolling back one
+        number_str = number_str[:-1]
+        self.index -= 1
+    elif (self.get_char_at() or "").isalpha():
+        # this was a string instead, sorry
+        self.index -= len(number_str)
+        return self.parse_string()
+    try:
+        if "," in number_str:
+            return str(number_str)
+        if "." in number_str or "e" in number_str or "E" in number_str:
+            return float(number_str)
+        else:
+            return int(number_str)
+    except ValueError:
+        return number_str

json_repair/parse_object.py

@@ -0,0 +1,110 @@
+from .constants import JSONReturnType
+from .json_context import ContextValues
+
+
+def parse_object(self) -> dict[str, JSONReturnType]:
+    # <object> ::= '{' [ <member> *(', ' <member>) ] '}' ; A sequence of 'members'
+    obj: dict[str, JSONReturnType] = {}
+    # Stop when you either find the closing parentheses or you have iterated over the entire string
+    while (self.get_char_at() or "}") != "}":
+        # This is what we expect to find:
+        # <member> ::= <string> ': ' <json>
+
+        # Skip filler whitespaces
+        self.skip_whitespaces_at()
+
+        # Sometimes LLMs do weird things, if we find a ":" so early, we'll change it to "," and move on
+        if (self.get_char_at() or "") == ":":
+            self.log(
+                "While parsing an object we found a : before a key, ignoring",
+            )
+            self.index += 1
+
+        # We are now searching for they string key
+        # Context is used in the string parser to manage the lack of quotes
+        self.context.set(ContextValues.OBJECT_KEY)
+
+        # Save this index in case we need find a duplicate key
+        rollback_index = self.index
+
+        # <member> starts with a <string>
+        key = ""
+        while self.get_char_at():
+            # The rollback index needs to be updated here in case the key is empty
+            rollback_index = self.index
+            if self.get_char_at() == "[" and key == "":
+                # Is this an array?
+                # Need to check if the previous parsed value contained in obj is an array and in that case parse and merge the two
+                prev_key = list(obj.keys())[-1] if obj else None
+                if prev_key and isinstance(obj[prev_key], list):
+                    # If the previous key's value is an array, parse the new array and merge
+                    self.index += 1
+                    new_array = self.parse_array()
+                    if isinstance(new_array, list):
+                        # Merge and flatten the arrays
+                        prev_value = obj[prev_key]
+                        if isinstance(prev_value, list):
+                            prev_value.extend(
+                                new_array[0] if len(new_array) == 1 and isinstance(new_array[0], list) else new_array
+                            )
+                        self.skip_whitespaces_at()
+                        if self.get_char_at() == ",":
+                            self.index += 1
+                        self.skip_whitespaces_at()
+                        continue
+            key = str(self.parse_string())
+            if key == "":
+                self.skip_whitespaces_at()
+            if key != "" or (key == "" and self.get_char_at() in [":", "}"]):
+                # If the string is empty but there is a object divider, we are done here
+                break
+        if ContextValues.ARRAY in self.context.context and key in obj:
+            self.log(
+                "While parsing an object we found a duplicate key, closing the object here and rolling back the index",
+            )
+            self.index = rollback_index - 1
+            # add an opening curly brace to make this work
+            self.json_str = self.json_str[: self.index + 1] + "{" + self.json_str[self.index + 1 :]
+            break
+
+        # Skip filler whitespaces
+        self.skip_whitespaces_at()
+
+        # We reached the end here
+        if (self.get_char_at() or "}") == "}":
+            continue
+
+        self.skip_whitespaces_at()
+
+        # An extreme case of missing ":" after a key
+        if (self.get_char_at() or "") != ":":
+            self.log(
+                "While parsing an object we missed a : after a key",
+            )
+
+        self.index += 1
+        self.context.reset()
+        self.context.set(ContextValues.OBJECT_VALUE)
+        # The value can be any valid json
+        self.skip_whitespaces_at()
+        # Corner case, a lone comma
+        value: JSONReturnType = ""
+        if (self.get_char_at() or "") in [",", "}"]:
+            self.log(
+                "While parsing an object value we found a stray , ignoring it",
+            )
+        else:
+            value = self.parse_json()
+
+        # Reset context since our job is done
+        self.context.reset()
+        obj[key] = value
+
+        if (self.get_char_at() or "") in [",", "'", '"']:
+            self.index += 1
+
+        # Remove trailing spaces
+        self.skip_whitespaces_at()
+
+    self.index += 1
+    return obj