json-repair 0.20.1__tar.gz → 0.22.0__tar.gz

{json_repair-0.20.1/src/json_repair.egg-info → json_repair-0.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: json_repair
-Version: 0.20.1
+Version: 0.22.0
 Summary: A package to repair broken json strings
 Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
 License: MIT License

{json_repair-0.20.1 → json_repair-0.22.0}/pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.0"]
 build-backend = "setuptools.build_meta"
 [project]
 name = "json_repair"
-version = "0.20.1"
+version = "0.22.0"
 license = {file = "LICENSE"}
 authors = [
   { name="Stefano Baccianella", email="4247706+mangiucugna@users.noreply.github.com" },

{json_repair-0.20.1 → json_repair-0.22.0}/src/json_repair/json_repair.py

@@ -1,7 +1,7 @@
 """
 This module will parse the JSON file following the BNF definition:
 
-    <json> ::= <primitive> | <container>
+    <json> ::= <container>
 
     <primitive> ::= <number> | <string> | <boolean>
     ; Where:
@@ -24,11 +24,55 @@ All supported use cases are in the unit tests
 
 import os
 import json
-from typing import Any, Dict, List, Union, TextIO
+from typing import Any, Dict, List, Optional, Union, TextIO, Tuple
+
+
+class StringFileWrapper:
+    # This is a trick to simplify the code, transform the filedescriptor handling into a string handling
+    def __init__(self, fd: TextIO) -> None:
+        self.fd = fd
+        self.length: int = 0
+
+    def __getitem__(self, index: int) -> str:
+        if isinstance(index, slice):
+            self.fd.seek(index.start)
+            value = self.fd.read(index.stop - index.start)
+            self.fd.seek(index.start)
+            return value
+        else:
+            self.fd.seek(index)
+            return self.fd.read(1)
+
+    def __len__(self) -> int:
+        if self.length < 1:
+            current_position = self.fd.tell()
+            self.fd.seek(0, os.SEEK_END)
+            self.length = self.fd.tell()
+            self.fd.seek(current_position)
+        return self.length
+
+    def __setitem__(self) -> None:
+        raise Exception("This is read-only!")
+
+
+class LoggerConfig:
+    # This is a type class to simplify the declaration
+    def __init__(self, log_level: Optional[str]):
+        self.log: List[Dict[str, str]] = []
+        self.window: int = 10
+        self.log_level: str = log_level if log_level else "none"
+
+
+JSONReturnType = Union[Dict[str, Any], List[Any], str, float, int, bool, None]
 
 
 class JSONParser:
-    def __init__(self, json_str: str, json_fd: TextIO, logging: bool = False) -> None:
+    def __init__(
+        self,
+        json_str: Union[str, StringFileWrapper],
+        json_fd: Optional[TextIO],
+        logging: Optional[bool],
+    ) -> None:
         # The string to parse
         self.json_str = json_str
         # Alternatively, the file description with a json file in it
@@ -36,26 +80,26 @@ class JSONParser:
             # This is a trick we do to treat the file wrapper as an array
             self.json_str = StringFileWrapper(json_fd)
         # Index is our iterator that will keep track of which character we are looking at right now
-        self.index = 0
+        self.index: int = 0
         # This is used in the object member parsing to manage the special cases of missing quotes in key or value
-        self.context = []
+        self.context: list[str] = []
         # Use this to log the activity, but only if logging is active
-        self.logger = {
-            "log": [],
-            "window": 10,
-            "log_level": "info" if logging else "none",
-        }
-
-    def parse(self) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
-        if self.logger["log_level"] == "none":
+        self.logger = LoggerConfig(log_level="info" if logging else None)
+
+    def parse(
+        self,
+    ) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
+        if self.logger.log_level == "none":
             return self.parse_json()
         else:
-            return self.parse_json(), self.logger["log"]
+            return self.parse_json(), self.logger.log
 
     def parse_json(
         self,
-    ) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
+    ) -> JSONReturnType:
         char = self.get_char_at()
+        # This parser will ignore any basic element (string or number) that is not inside an array or object
+        is_in_context = len(self.context) > 0
         # False means that we are at the end of the string provided, is the base case for recursion
         if char is False:
             return ""
@@ -78,10 +122,10 @@ class JSONParser:
             )
             return ""
         # <string> starts with a quote
-        elif char in ['"', "'", "“"] or char.isalpha():
+        elif is_in_context and (char in ['"', "'", "“"] or char.isalpha()):
             return self.parse_string()
         # <number> starts with [0-9] or minus
-        elif char.isdigit() or char == "-" or char == ".":
+        elif is_in_context and (char.isdigit() or char == "-" or char == "."):
             return self.parse_number()
         # If everything else fails, we just ignore and move on
         else:
@@ -225,7 +269,7 @@ class JSONParser:
         self.reset_context()
         return arr
 
-    def parse_string(self) -> str:
+    def parse_string(self) -> Union[str, JSONReturnType]:
         # <string> is a string of valid characters enclosed in quotes
         # i.e. { name: "John" }
         # Somehow all weird cases in an invalid JSON happen to be resolved in this function, so be careful here
@@ -324,7 +368,7 @@ class JSONParser:
                 string_acc = string_acc[:-1]
                 if char in [rstring_delimiter, "t", "n", "r", "b", "\\"]:
                     escape_seqs = {"t": "\t", "n": "\n", "r": "\r", "b": "\b"}
-                    string_acc += escape_seqs.get(char, char)
+                    string_acc += escape_seqs.get(char, char) or char
                     self.index += 1
                     char = self.get_char_at()
             # ChatGPT sometimes forget to quote stuff in html tags or markdown, so we do this whole thing here
@@ -362,7 +406,29 @@ class JSONParser:
                             break
                         i += 1
                         next_c = self.get_char_at(i)
-                    if next_c == rstring_delimiter:
+                    # If we stopped for a comma in object_value context, let's check if find a "} at the end of the string
+                    if next_c == "," and self.get_context() == "object_value":
+                        i += 1
+                        next_c = self.get_char_at(i)
+                        while next_c and next_c != rstring_delimiter:
+                            i += 1
+                            next_c = self.get_char_at(i)
+                        # Ok now I found a delimiter, let's skip whitespaces and see if next we find a }
+                        i += 1
+                        next_c = self.get_char_at(i)
+                        while next_c and next_c.isspace():
+                            i += 1
+                            next_c = self.get_char_at(i)
+                        if next_c == "}":
+                            # OK this is valid then
+                            self.log(
+                                "While parsing a string, we a misplaced quote that would have closed the string but has a different meaning here since this is the last element of the object, ignoring it",
+                                "info",
+                            )
+                            string_acc += char
+                            self.index += 1
+                            char = self.get_char_at()
+                    elif next_c == rstring_delimiter:
                         if self.get_context() == "object_value":
                             # But this might not be it! This could be just a missing comma
                             # We found a delimiter and we need to check if this is a key
@@ -418,7 +484,7 @@ class JSONParser:
 
         return string_acc.rstrip()
 
-    def parse_number(self) -> Union[float, int, str]:
+    def parse_number(self) -> Union[float, int, str, JSONReturnType]:
         # <number> is a valid real number expressed in one of a number of given formats
         number_str = ""
         number_chars = set("0123456789-.eE/,")
@@ -451,8 +517,7 @@ class JSONParser:
     def parse_boolean_or_null(self) -> Union[bool, str, None]:
         # <boolean> is one of the literal strings 'true', 'false', or 'null' (unquoted)
         starting_index = self.index
-        value = ""
-        char = self.get_char_at().lower()
+        char = (self.get_char_at() or "").lower()
         if char == "t":
             value = ("true", True)
         elif char == "f":
@@ -460,12 +525,12 @@ class JSONParser:
         elif char == "n":
             value = ("null", None)
 
-        if len(value):
+        if value:
             i = 0
             while char and i < len(value[0]) and char == value[0][i]:
                 i += 1
                 self.index += 1
-                char = self.get_char_at().lower()
+                char = (self.get_char_at() or "").lower()
             if i == len(value[0]):
                 return value[1]
 
@@ -513,12 +578,12 @@ class JSONParser:
             return ""
 
     def log(self, text: str, level: str) -> None:
-        if level == self.logger["log_level"]:
+        if level == self.logger.log_level:
             context = ""
-            start = max(self.index - self.logger["window"], 0)
-            end = min(self.index + self.logger["window"], len(self.json_str))
+            start = max(self.index - self.logger.window, 0)
+            end = min(self.index + self.logger.window, len(self.json_str))
             context = self.json_str[start:end]
-            self.logger["log"].append(
+            self.logger.log.append(
                 {
                     "text": text,
                     "context": context,
@@ -528,11 +593,11 @@ class JSONParser:
 
 def repair_json(
     json_str: str = "",
-    return_objects: bool = False,
-    skip_json_loads: bool = False,
-    logging: bool = False,
-    json_fd: TextIO = None,
-) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
+    return_objects: Optional[bool] = False,
+    skip_json_loads: Optional[bool] = False,
+    logging: Optional[bool] = False,
+    json_fd: Optional[TextIO] = None,
+) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
     It will return the fixed string by default.
@@ -559,7 +624,7 @@ def repair_json(
 
 def loads(
     json_str: str, skip_json_loads: bool = False, logging: bool = False
-) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
+) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     This function works like `json.loads()` except that it will fix your JSON in the process.
     It is a wrapper around the `repair_json()` function with `return_objects=True`.
@@ -574,7 +639,7 @@ def loads(
 
 def load(
     fd: TextIO, skip_json_loads: bool = False, logging: bool = False
-) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
+) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     This function works like `json.load()` except that it will fix your JSON in the process.
     It is a wrapper around the `repair_json()` function with `json_fd=fd` and `return_objects=True`.
@@ -584,7 +649,7 @@ def load(
 
 def from_file(
     filename: str, skip_json_loads: bool = False, logging: bool = False
-) -> Union[Dict[str, Any], List[Any], str, float, int, bool, None]:
+) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     This function is a wrapper around `load()` so you can pass the filename as string
     """
@@ -593,31 +658,3 @@ def from_file(
     fd.close()
 
     return jsonobj
-
-
-class StringFileWrapper:
-    # This is a trick to simplify the code above, transform the filedescriptor handling into an array handling
-    def __init__(self, fd: TextIO) -> None:
-        self.fd = fd
-        self.length = None
-
-    def __getitem__(self, index: int) -> Any:
-        if isinstance(index, slice):
-            self.fd.seek(index.start)
-            value = self.fd.read(index.stop - index.start)
-            self.fd.seek(index.start)
-            return value
-        else:
-            self.fd.seek(index)
-            return self.fd.read(1)
-
-    def __len__(self) -> int:
-        if not self.length:
-            current_position = self.fd.tell()
-            self.fd.seek(0, os.SEEK_END)
-            self.length = self.fd.tell()
-            self.fd.seek(current_position)
-        return self.length
-
-    def __setitem__(self):
-        raise Exception("This is read-only!")

{json_repair-0.20.1 → json_repair-0.22.0/src/json_repair.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: json_repair
-Version: 0.20.1
+Version: 0.22.0
 Summary: A package to repair broken json strings
 Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
 License: MIT License

{json_repair-0.20.1 → json_repair-0.22.0}/tests/test_json_repair.py

@@ -1,9 +1,26 @@
 from src.json_repair.json_repair import from_file, repair_json, loads
 
+def test_basic_types_valid():
+    assert repair_json("True", return_objects=True) == ""
+    assert repair_json("False", return_objects=True) == ""
+    assert repair_json("Null", return_objects=True) == ""
+    assert repair_json("1", return_objects=True) == 1
+    assert repair_json("[]", return_objects=True) == []
+    assert repair_json("[1, 2, 3, 4]", return_objects=True) == [1, 2, 3, 4]
+    assert repair_json("{}", return_objects=True) == {}
+    assert repair_json('{ "key": "value", "key2": 1, "key3": True }', return_objects=True) == { "key": "value", "key2": 1, "key3": True }
+
+def test_basic_types_invalid():
+    assert repair_json("true", return_objects=True) == True
+    assert repair_json("false", return_objects=True) == False
+    assert repair_json("null", return_objects=True) == None
+    assert repair_json("1.2", return_objects=True) == 1.2
+    assert repair_json("[", return_objects=True) == []
+    assert repair_json("[1, 2, 3, 4", return_objects=True) == [1, 2, 3, 4]
+    assert repair_json("{", return_objects=True) == {}
+    assert repair_json('{ "key": value, "key2": 1 "key3": null }', return_objects=True) == { "key": "value", "key2": 1, "key3": None }
 
 def test_valid_json():
-    assert repair_json("[]") == "[]"
-    assert repair_json("[1, 2, 3, 4]") == "[1, 2, 3, 4]"
     assert (
         repair_json('{"name": "John", "age": 30, "city": "New York"}')
         == '{"name": "John", "age": 30, "city": "New York"}'
@@ -77,6 +94,7 @@ def test_missing_and_mixed_quotes():
         repair_json('{"name": "John", "age": 30, "city": "New')
         == '{"name": "John", "age": 30, "city": "New"}'
     )
+    assert repair_json('[{"key": "value", COMMENT "notes": "lorem "ipsum", sic."}]') == '[{"key": "value", "notes": "lorem \\"ipsum\\", sic."}]'
 
 def test_array_edge_cases():
     assert repair_json("[1, 2, 3,") == "[1, 2, 3]"
@@ -96,7 +114,7 @@ def test_array_edge_cases():
 
     
 def test_escaping():
-    assert repair_json("'\"'") == '"\\\""'
+    assert repair_json("'\"'") == '""'
     assert repair_json("{\"key\": 'string\"\n\t\le'") == '{"key": "string\\"\\n\\tle"}'
     assert repair_json(r'{"real_content": "Some string: Some other string \t Some string <a href=\"https://domain.com\">Some link</a>"') == r'{"real_content": "Some string: Some other string \t Some string <a href=\"https://domain.com\">Some link</a>"}'
     assert repair_json('{"key_1\n": "value"}') == '{"key_1": "value"}'
@@ -146,6 +164,7 @@ def test_leading_trailing_characters():
     assert repair_json("""{    "a": "",    "b": [ { "c": 1} ] \n}```""") == '{"a": "", "b": [{"c": 1}]}'
     assert repair_json("Based on the information extracted, here is the filled JSON output: ```json { 'a': 'b' } ```") == '{"a": "b"}'
     assert repair_json("""
+                       The next 64 elements are:
                        ```json
                        { "key": "value" }
                        ```""") == '{"key": "value"}'

{json_repair-0.20.1 → json_repair-0.22.0}/tests/test_performance.py

@@ -19,7 +19,7 @@ def test_true_true_correct(benchmark):
   mean_time = benchmark.stats.get("median")
 
   # Define your time threshold in seconds
-  max_time = 13 / 10 ** 4  # 1.3 millisecond
+  max_time = 14 / 10 ** 4  # 1.4 millisecond
 
   # Assert that the average time is below the threshold
   assert mean_time < max_time, f"Benchmark exceeded threshold: {mean_time:.3f}s > {max_time:.3f}s"
@@ -31,7 +31,7 @@ def test_true_true_incorrect(benchmark):
   mean_time = benchmark.stats.get("median")
 
   # Define your time threshold in seconds
-  max_time = 13 / 10 ** 4  # 1.3 millisecond
+  max_time = 14 / 10 ** 4  # 1.4 millisecond
 
   # Assert that the average time is below the threshold
   assert mean_time < max_time, f"Benchmark exceeded threshold: {mean_time:.3f}s > {max_time:.3f}s"
@@ -53,7 +53,7 @@ def test_true_false_incorrect(benchmark):
   mean_time = benchmark.stats.get("median")
 
   # Define your time threshold in seconds
-  max_time = 13 / 10 ** 4  # 1.3 millisecond
+  max_time = 14 / 10 ** 4  # 1.4 millisecond
 
   # Assert that the average time is below the threshold
   assert mean_time < max_time, f"Benchmark exceeded threshold: {mean_time:.3f}s > {max_time:.3f}s"