json-repair 0.28.3__py3-none-any.whl → 0.29.0__py3-none-any.whl

json_repair/__main__.py

@@ -0,0 +1,4 @@
+from .json_repair import cli
+
+if __name__ == "__main__":
+    cli()

json_repair/json_repair.py

@@ -22,26 +22,62 @@ If something is wrong (a missing parantheses or quotes for example) it will use
 All supported use cases are in the unit tests
 """
 
+import argparse
 import os
+import sys
 import json
 from typing import Any, Dict, List, Optional, Union, TextIO, Tuple, Literal
 
 
 class StringFileWrapper:
     # This is a trick to simplify the code, transform the filedescriptor handling into a string handling
-    def __init__(self, fd: TextIO) -> None:
+    def __init__(self, fd: TextIO, CHUNK_LENGTH: int) -> None:
         self.fd = fd
         self.length: int = 0
+        # Buffers are 1MB strings that are read from the file
+        # and kept in memory to keep reads low
+        self.buffers: dict[int, str] = {}
+        # CHUNK_LENGTH is in bytes
+        if not CHUNK_LENGTH or CHUNK_LENGTH < 2:
+            CHUNK_LENGTH = 1_000_000
+        self.buffer_length = CHUNK_LENGTH
+
+    def get_buffer(self, index: int) -> str:
+        if self.buffers.get(index) is None:
+            self.fd.seek(index * self.buffer_length)
+            self.buffers[index] = self.fd.read(self.buffer_length)
+            # Save memory by keeping max 2MB buffer chunks and min 2 chunks
+            if len(self.buffers) > max(2, 2_000_000 / self.buffer_length):
+                oldest_key = next(iter(self.buffers))
+                if oldest_key != index:
+                    self.buffers.pop(oldest_key)
+        return self.buffers[index]
 
     def __getitem__(self, index: Union[int, slice]) -> str:
+        # The buffer is an array that is seek like a RAM:
+        # self.buffers[index]: the row in the array of length 1MB, index is `i` modulo CHUNK_LENGTH
+        # self.buffures[index][j]: the column of the row that is `i` remainder CHUNK_LENGTH
         if isinstance(index, slice):
-            self.fd.seek(index.start)
-            value = self.fd.read(index.stop - index.start)
-            self.fd.seek(index.start)
-            return value
+            buffer_index = index.start // self.buffer_length
+            buffer_end = index.stop // self.buffer_length
+            if buffer_index == buffer_end:
+                return self.get_buffer(buffer_index)[
+                    index.start % self.buffer_length : index.stop % self.buffer_length
+                ]
+            else:
+                start_slice = self.get_buffer(buffer_index)[
+                    index.start % self.buffer_length :
+                ]
+                end_slice = self.get_buffer(buffer_end)[
+                    : index.stop % self.buffer_length
+                ]
+                middle_slices = [
+                    self.get_buffer(i) for i in range(buffer_index + 1, buffer_end)
+                ]
+                return start_slice + "".join(middle_slices) + end_slice
         else:
-            self.fd.seek(index)
-            return self.fd.read(1)
+            buffer_index = index // self.buffer_length
+            return self.get_buffer(buffer_index)[index % self.buffer_length]
 
     def __len__(self) -> int:
         if self.length < 1:
@@ -69,13 +105,14 @@ class JSONParser:
         json_str: Union[str, StringFileWrapper],
         json_fd: Optional[TextIO],
         logging: Optional[bool],
+        json_fd_chunk_length: int = 0,
     ) -> None:
         # The string to parse
         self.json_str = json_str
         # Alternatively, the file description with a json file in it
         if json_fd:
             # This is a trick we do to treat the file wrapper as an array
-            self.json_str = StringFileWrapper(json_fd)
+            self.json_str = StringFileWrapper(json_fd, json_fd_chunk_length)
         # Index is our iterator that will keep track of which character we are looking at right now
         self.index: int = 0
         # This is used in the object member parsing to manage the special cases of missing quotes in key or value
@@ -639,6 +676,7 @@ def repair_json(
     logging: bool = False,
     json_fd: Optional[TextIO] = None,
     ensure_ascii: bool = True,
+    chunk_length: int = 0,
 ) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     Given a json formatted string, it will try to decode it and, if it fails, it will try to fix it.
@@ -647,7 +685,7 @@ def repair_json(
     When `skip_json_loads=True` is passed, it will not call the built-in json.loads() function
     When `logging=True` is passed, it will return a tuple with the repaired json and a log of all repair actions
     """
-    parser = JSONParser(json_str, json_fd, logging)
+    parser = JSONParser(json_str, json_fd, logging, chunk_length)
     if skip_json_loads:
         parsed_json = parser.parse()
     else:
@@ -683,7 +721,10 @@ def loads(
 
 
 def load(
-    fd: TextIO, skip_json_loads: bool = False, logging: bool = False
+    fd: TextIO,
+    skip_json_loads: bool = False,
+    logging: bool = False,
+    chunk_length: int = 0,
 ) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     This function works like `json.load()` except that it will fix your JSON in the process.
@@ -691,6 +732,7 @@ def load(
     """
     return repair_json(
         json_fd=fd,
+        chunk_length=chunk_length,
         return_objects=True,
         skip_json_loads=skip_json_loads,
         logging=logging,
@@ -701,12 +743,62 @@ def from_file(
     filename: str,
     skip_json_loads: bool = False,
     logging: bool = False,
+    chunk_length: int = 0,
 ) -> Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]:
     """
     This function is a wrapper around `load()` so you can pass the filename as string
     """
     fd = open(filename)
-    jsonobj = load(fd, skip_json_loads, logging)
+    jsonobj = load(
+        fd=fd,
+        skip_json_loads=skip_json_loads,
+        logging=logging,
+        chunk_length=chunk_length,
+    )
     fd.close()
 
     return jsonobj
+
+
+def cli():  # pragma: no cover
+    parser = argparse.ArgumentParser(description="Repair and parse JSON files.")
+    parser.add_argument("filename", help="The JSON file to repair")
+    parser.add_argument(
+        "-i",
+        "--inline",
+        action="store_true",
+        help="Replace the file inline instead of returning the output to stdout",
+    )
+    parser.add_argument(
+        "--ensure_ascii",
+        action="store_true",
+        help="Pass the ensure_ascii parameter to json.dumps()",
+    )
+    parser.add_argument(
+        "--indent",
+        type=int,
+        default=2,
+        help="Number of spaces for indentation (Default 2)",
+    )
+
+    args = parser.parse_args()
+
+    ensure_ascii = False
+    if args.ensure_ascii:
+        ensure_ascii = True
+    try:
+        result = from_file(args.filename)
+
+        if args.inline:
+            fd = open(args.filename, mode="w")
+            json.dump(result, fd, indent=args.indent, ensure_ascii=ensure_ascii)
+            fd.close()
+        else:
+            print(json.dumps(result, indent=args.indent, ensure_ascii=ensure_ascii))
+    except Exception as e:
+        print(f"Error: {str(e)}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    cli()

{json_repair-0.28.3.dist-info → json_repair-0.29.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: json_repair
-Version: 0.28.3
+Version: 0.29.0
 Summary: A package to repair broken json strings
 Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
 License: MIT License
@@ -45,6 +45,21 @@ This simple package can be used to fix an invalid json string. To know all cases
 
 Inspired by https://github.com/josdejong/jsonrepair
 
+---
+# How to cite
+If you are using this library in your academic work (as I know many folks are) please find the BibTex here:
+
+    @software{Baccianella_JSON_Repair_-_2024,
+        author = {Baccianella, Stefano},
+        month = aug,
+        title = {{JSON Repair - A python module to repair invalid JSON, commonly used to parse the output of LLMs}},
+        url = {https://github.com/mangiucugna/json_repair},
+        version = {0.28.3},
+        year = {2024}
+    }
+
+Thank you for citing my work and please send me a link to the paper if you can!
+
 ---
 # Offer me a beer
 If you find this library useful, you can help me by donating toward my monthly beer budget here: https://github.com/sponsors/mangiucugna
@@ -82,6 +97,18 @@ or just
 
     decoded_object = json_repair.repair_json(json_string, return_objects=True)
 
+### Avoid this antipattern
+Some users of this library adopt the following pattern:
+
+    obj = {}
+    try:
+        obj = json.loads(string)
+    except json.JSONDecodeError as e:
+        obj = json_repair.loads(string)
+        ...
+
+This is wasteful because `json_repair` will already verify for you if the JSON is valid, if you still want to do that then add `skip_json_loads=True` to the call as explained the section below.
+
 ### Read json from a file or file descriptor
 
 JSON repair provides also a drop-in replacement for `json.load()`:
@@ -122,6 +149,32 @@ Some rules of thumb to use:
 - Setting `return_objects=True` will always be faster because the parser returns an object already and it doesn't have serialize that object to JSON
 - `skip_json_loads` is faster only if you 100% know that the string is not a valid JSON
 - If you are having issues with escaping pass the string as **raw** string like: `r"string with escaping\""`
+
+### Use json_repair from CLI
+
+Install the library for command-line with:
+```
+pipx install json-repair
+```
+then run
+```
+$ json_repair -h
+
+usage: json_repair [-h] [-i] [--ensure_ascii] [--indent INDENT] filename
+
+Repair and parse JSON files.
+
+positional arguments:
+  filename         The JSON file to repair
+
+options:
+  -h, --help       show this help message and exit
+  -i, --inline     Replace the file inline instead of returning the output to stdout
+  --ensure_ascii   Pass the ensure_ascii parameter to json.dumps()
+  --indent INDENT  Number of spaces for indentation (Default 2)
+```
+to learn how to use it
+
 ## Adding to requirements
 **Please pin this library only on the major version!**
 

json_repair-0.29.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+json_repair/__init__.py,sha256=IIzSm1DsCRrr8seF3UeMZXwxcq-tE3j-8d1WBxvEJvE,178
+json_repair/__main__.py,sha256=EsJb-y89uZEvGQQg1GdIDWzfDwfOMvVekKEtdguQXCM,67
+json_repair/json_repair.py,sha256=hltJ3Qa4qFbUD3mVKkYvFWksnCcIZqx8zamKfBpjeNs,33538
+json_repair/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+json_repair-0.29.0.dist-info/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
+json_repair-0.29.0.dist-info/METADATA,sha256=yh0EJo-I1u0R6X-Gq9ETz0WbgmuGIhzR7Icw9W4Kee0,9630
+json_repair-0.29.0.dist-info/WHEEL,sha256=uCRv0ZEik_232NlR4YDw4Pv3Ajt5bKvMH13NUU7hFuI,91
+json_repair-0.29.0.dist-info/entry_points.txt,sha256=SNfge3zPSP-ASqriYU9r3NAPaXdseYr7ciPMKdV2uSw,57
+json_repair-0.29.0.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
+json_repair-0.29.0.dist-info/RECORD,,

{json_repair-0.28.3.dist-info → json_repair-0.29.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (72.2.0)
+Generator: setuptools (74.1.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

json_repair-0.29.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+json_repair = json_repair.__main__:cli

json_repair-0.28.3.dist-info/RECORD

@@ -1,8 +0,0 @@
-json_repair/__init__.py,sha256=IIzSm1DsCRrr8seF3UeMZXwxcq-tE3j-8d1WBxvEJvE,178
-json_repair/json_repair.py,sha256=QShXijcgNG3ejW_rBbmk0RMjJE1KlGvYBhXcQnMZcHo,30129
-json_repair/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-json_repair-0.28.3.dist-info/LICENSE,sha256=wrjQo8MhNrNCicXtMe3MHmS-fx8AmQk1ue8AQwiiFV8,1076
-json_repair-0.28.3.dist-info/METADATA,sha256=ZIm82pnDJX68089RoprQJq-HrL2LF1LVr4xDTh_6VJI,8043
-json_repair-0.28.3.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
-json_repair-0.28.3.dist-info/top_level.txt,sha256=7-VZwZN2CgB_n0NlSLk-rEUFh8ug21lESbsblOYuZqw,12
-json_repair-0.28.3.dist-info/RECORD,,