json-repair 0.55.1__py3-none-any.whl → 0.56.0__py3-none-any.whl

json_repair/json_parser.py

@@ -1,4 +1,5 @@
-from typing import TextIO
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any, TextIO
 
 from .parse_array import parse_array as _parse_array
 from .parse_comment import parse_comment as _parse_comment
@@ -10,11 +11,18 @@ from .utils.json_context import JsonContext
 from .utils.object_comparer import ObjectComparer
 from .utils.string_file_wrapper import StringFileWrapper
 
+if TYPE_CHECKING:
+    from .schema_repair import SchemaRepairer
+
 
 class JSONParser:
     # Split the parse methods into separate files because this one was like 3000 lines
-    def parse_array(self) -> list[JSONReturnType]:
-        return _parse_array(self)
+    def parse_array(
+        self,
+        schema: dict[str, Any] | bool | None = None,
+        path: str = "$",
+    ) -> list[JSONReturnType]:
+        return _parse_array(self, schema, path)
 
     def parse_comment(self) -> JSONReturnType:
         return _parse_comment(self)
@@ -22,8 +30,12 @@ class JSONParser:
     def parse_number(self) -> JSONReturnType:
         return _parse_number(self)
 
-    def parse_object(self) -> JSONReturnType:
-        return _parse_object(self)
+    def parse_object(
+        self,
+        schema: dict[str, Any] | bool | None = None,
+        path: str = "$",
+    ) -> JSONReturnType:
+        return _parse_object(self, schema, path)
 
     def parse_string(self) -> JSONReturnType:
         return _parse_string(self)
@@ -53,8 +65,8 @@ class JSONParser:
         # We could add a guard in the code for each call but that would make this code unreadable, so here's this neat trick
         # Replace self.log with a noop
         self.logging = logging
+        self.logger: list[dict[str, str]] = []
         if logging:
-            self.logger: list[dict[str, str]] = []
             self.log = self._log
         else:
             # No-op
@@ -71,11 +83,26 @@ class JSONParser:
         # may not be desirable in some use cases and the user would prefer json_repair to return an exception.
         # So strict mode was added to disable some of those heuristics.
         self.strict = strict
+        self.schema_repairer: SchemaRepairer | None = None
 
     def parse(
         self,
-    ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
-        json = self.parse_json()
+    ) -> JSONReturnType:
+        return self._parse_top_level(self.parse_json)
+
+    def parse_with_schema(
+        self,
+        repairer: "SchemaRepairer",
+        schema: dict[str, Any] | bool,
+    ) -> JSONReturnType:
+        """Parse with schema guidance enabled for all nested values."""
+        self.schema_repairer = repairer
+        return self._parse_top_level(lambda: self.parse_json(schema, "$"))
+
+    # Consolidate top-level parsing so we handle multiple sequential JSON values consistently
+    # (including update semantics and strict-mode validation).
+    def _parse_top_level(self, parse_element: Callable[[], JSONReturnType]) -> JSONReturnType:
+        json = parse_element()
         if self.index < len(self.json_str):
             self.log(
                 "The parser returned early, checking if there's more json elements",
@@ -83,19 +110,17 @@ class JSONParser:
             json = [json]
             while self.index < len(self.json_str):
                 self.context.reset()
-                j = self.parse_json()
+                j = parse_element()
                 if j:
                     if ObjectComparer.is_same_object(json[-1], j):
-                        # replace the last entry with the new one since the new one seems an update
+                        # Treat repeated objects as updates: keep the newest value.
                         json.pop()
                     else:
                         if not json[-1]:
                             json.pop()
                     json.append(j)
                 else:
-                    # this was a bust, move the index
                     self.index += 1
-            # If nothing extra was found, don't return an array
             if len(json) == 1:
                 self.log(
                     "There were no more elements, returning the element without the array",
@@ -106,38 +131,51 @@ class JSONParser:
                     "Multiple top-level JSON elements found in strict mode, raising an error",
                 )
                 raise ValueError("Multiple top-level JSON elements found in strict mode.")
-        if self.logging:
-            return json, self.logger
-        else:
-            return json
+        return json
 
     def parse_json(
         self,
+        schema: dict[str, Any] | bool | None = None,
+        path: str = "$",
     ) -> JSONReturnType:
+        """Parse the next JSON value and, when configured, enforce schema constraints."""
+        repairer = self.schema_repairer if self.schema_repairer is not None and schema not in (None, True) else None
+        if repairer is not None:
+            # Resolve references once and decide whether schema-guided repairs are needed.
+            schema = repairer.resolve_schema(schema)
+            if schema is True:
+                repairer = None
+            elif schema is False:
+                raise ValueError("Schema does not allow any values.")
+
         while True:
             char = self.get_char_at()
             # None means that we are at the end of the string provided
             if char is None:
                 return ""
             # <object> starts with '{'
-            elif char == "{":
+            if char == "{":
                 self.index += 1
-                return self.parse_object()
+                value = self.parse_object(schema, path) if repairer else self.parse_object()
+                return repairer.repair_value(value, schema, path) if repairer else value
             # <array> starts with '['
-            elif char == "[":
+            if char == "[":
                 self.index += 1
-                return self.parse_array()
+                value = self.parse_array(schema, path) if repairer else self.parse_array()
+                return repairer.repair_value(value, schema, path) if repairer else value
             # <string> starts with a quote
-            elif not self.context.empty and (char in STRING_DELIMITERS or char.isalpha()):
-                return self.parse_string()
+            if not self.context.empty and (char in STRING_DELIMITERS or char.isalpha()):
+                value = self.parse_string()
+                return repairer.repair_value(value, schema, path) if repairer else value
             # <number> starts with [0-9] or minus
-            elif not self.context.empty and (char.isdigit() or char == "-" or char == "."):
-                return self.parse_number()
-            elif char in ["#", "/"]:
-                return self.parse_comment()
+            if not self.context.empty and (char.isdigit() or char == "-" or char == "."):
+                value = self.parse_number()
+                return repairer.repair_value(value, schema, path) if repairer else value
+            if char in ["#", "/"]:
+                value = self.parse_comment()
+                return repairer.repair_value(value, schema, path) if repairer else value
             # If everything else fails, we just ignore and move on
-            else:
-                self.index += 1
+            self.index += 1
 
     def get_char_at(self, count: int = 0) -> str | None:
         # Why not use something simpler? Because try/except in python is a faster alternative to an "if" statement that is often True

json_repair/json_repair.py

@@ -25,9 +25,11 @@ All supported use cases are in the unit tests
 import argparse
 import json
 import sys
+from pathlib import Path
 from typing import Any, Literal, TextIO, overload
 
 from .json_parser import JSONParser
+from .schema_repair import SchemaRepairer, load_schema_model, schema_from_input
 from .utils.constants import JSONReturnType
 
 
@@ -41,6 +43,7 @@ def repair_json(
     chunk_length: int = 0,
     stream_stable: bool = False,
     strict: bool = False,
+    schema: Any | None = None,
     **json_dumps_args: Any,
 ) -> str: ...
 
@@ -55,6 +58,7 @@ def repair_json(
     chunk_length: int = 0,
     stream_stable: bool = False,
     strict: bool = False,
+    schema: Any | None = None,
     **json_dumps_args: Any,
 ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: ...
 
@@ -68,6 +72,7 @@ def repair_json(
     chunk_length: int = 0,
     stream_stable: bool = False,
     strict: bool = False,
+    schema: Any | None = None,
     **json_dumps_args: Any,
 ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
     """
@@ -83,27 +88,49 @@ def repair_json(
         chunk_length (int, optional): Size in bytes of the file chunks to read at once. Ignored if `json_fd` is None. Do not use! Use `from_file` or `load` instead. Defaults to 1MB.
         stream_stable (bool, optional): When the json to be repaired is the accumulation of streaming json at a certain moment.If this parameter to True will keep the repair results stable.
         strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
+        schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs. Schema guidance is skipped for already-valid JSON unless `skip_json_loads=True`.
     Returns:
         Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON or a tuple with the repaired JSON and repair log when logging is True.
     """
+    # Schema-guided repairs and strict mode are mutually exclusive to avoid conflicting behavior.
+    if schema is not None and strict:
+        raise ValueError("schema and strict cannot be used together.")
+
     parser = JSONParser(json_str, json_fd, logging, chunk_length, stream_stable, strict)
-    if skip_json_loads:
-        parsed_json = parser.parse()
-    else:
+    # When JSON is already valid, skip schema guidance unless the caller explicitly disables json.loads.
+    if not skip_json_loads:
+        loaded_json: JSONReturnType | None
         try:
-            parsed_json = json.load(json_fd) if json_fd else json.loads(json_str)
+            loaded_json = json.load(json_fd) if json_fd else json.loads(json_str)
         except json.JSONDecodeError:
-            parsed_json = parser.parse()
+            loaded_json = None
+        else:
+            if logging:
+                return loaded_json, []
+            if return_objects:
+                return loaded_json
+            if loaded_json == "":
+                return ""
+            return json.dumps(loaded_json, **json_dumps_args)
+
+    # Schema guidance only happens in parser mode.
+    schema_obj = schema_from_input(schema) if schema is not None else None
+    parsed_json: JSONReturnType
+    if schema_obj is None:
+        parsed_json = parser.parse()
+    else:
+        repairer = SchemaRepairer(schema_obj, parser.logger if logging else None)
+        parsed_json = parser.parse_with_schema(repairer, schema_obj)
+        # Post-parse validation ensures we reject values that cannot satisfy the schema.
+        repairer.validate(parsed_json, schema_obj)
     # It's useful to return the actual object instead of the json string,
     # it allows this lib to be a replacement of the json library
-    if return_objects or logging:
-        # If logging is True, the user should expect a tuple.
-        # If json.load(s) worked, the repair log list is empty
-        if logging and not isinstance(parsed_json, tuple):
-            return parsed_json, []
+    if logging:
+        return parsed_json, parser.logger
+    if return_objects:
         return parsed_json
     # Avoid returning only a pair of quotes if it's an empty string
-    elif parsed_json == "":
+    if parsed_json == "":
         return ""
     return json.dumps(parsed_json, **json_dumps_args)
 
@@ -114,6 +141,7 @@ def loads(
     logging: bool = False,
     stream_stable: bool = False,
     strict: bool = False,
+    schema: Any | None = None,
 ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]] | str:
     """
     This function works like `json.loads()` except that it will fix your JSON in the process.
@@ -124,6 +152,7 @@ def loads(
         skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
         logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
         strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
+        schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs. Schema guidance is skipped for already-valid JSON unless `skip_json_loads=True`.
 
     Returns:
         Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]], str]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
@@ -135,6 +164,7 @@ def loads(
         logging=logging,
         stream_stable=stream_stable,
         strict=strict,
+        schema=schema,
     )
 
 
@@ -144,6 +174,7 @@ def load(
     logging: bool = False,
     chunk_length: int = 0,
     strict: bool = False,
+    schema: Any | None = None,
 ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
     """
     This function works like `json.load()` except that it will fix your JSON in the process.
@@ -155,6 +186,7 @@ def load(
         logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
         chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
         strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
+        schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs. Schema guidance is skipped for already-valid JSON unless `skip_json_loads=True`.
 
     Returns:
         Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
@@ -166,40 +198,42 @@ def load(
         skip_json_loads=skip_json_loads,
         logging=logging,
         strict=strict,
+        schema=schema,
     )
 
 
 def from_file(
-    filename: str,
+    filename: str | Path,
     skip_json_loads: bool = False,
     logging: bool = False,
     chunk_length: int = 0,
     strict: bool = False,
+    schema: Any | None = None,
 ) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
     """
     This function is a wrapper around `load()` so you can pass the filename as string
 
     Args:
-        filename (str): The name of the file containing JSON data to load and repair.
+        filename (str | Path): The name of the file containing JSON data to load and repair.
         skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
         logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
         chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
         strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
+        schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs. Schema guidance is skipped for already-valid JSON unless `skip_json_loads=True`.
 
     Returns:
         Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
     """
-    with open(filename) as fd:
-        jsonobj = load(
+    with Path(filename).open() as fd:
+        return load(
             fd=fd,
             skip_json_loads=skip_json_loads,
             logging=logging,
             chunk_length=chunk_length,
             strict=strict,
+            schema=schema,
         )
 
-    return jsonobj
-
 
 def cli(inline_args: list[str] | None = None) -> int:
     """
@@ -212,6 +246,10 @@ def cli(inline_args: list[str] | None = None) -> int:
             - -o, --output TARGET (str): If specified, the output will be written to TARGET filename instead of stdout.
             - --ensure_ascii (bool): Pass ensure_ascii=True to json.dumps(). Will pass False otherwise.
             - --indent INDENT (int): Number of spaces for indentation (Default 2).
+            - --skip-json-loads (bool): Skip initial json.loads validation (needed to force schema on valid JSON).
+            - --schema SCHEMA (str): Path to a JSON Schema file that guides repairs.
+            - --schema-model MODEL (str): Pydantic v2 model in 'module:ClassName' form that guides repairs.
+            - --strict (bool): Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them.
 
     Returns:
         int: Exit code of the CLI operation.
@@ -253,13 +291,28 @@ def cli(inline_args: list[str] | None = None) -> int:
         default=2,
         help="Number of spaces for indentation (Default 2)",
     )
+    parser.add_argument(
+        "--skip-json-loads",
+        action="store_true",
+        help="Skip initial json.loads validation (needed to force schema on valid JSON)",
+    )
+    parser.add_argument(
+        "--schema",
+        metavar="SCHEMA",
+        help="Path to a JSON Schema file that guides repairs",
+    )
+    parser.add_argument(
+        "--schema-model",
+        metavar="MODEL",
+        help="Pydantic v2 model in 'module:ClassName' form that guides repairs",
+    )
     parser.add_argument(
         "--strict",
         action="store_true",
         help="Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them",
     )
 
-    args = parser.parse_args() if inline_args is None else parser.parse_args(inline_args)
+    args = parser.parse_args(inline_args)
 
     # Inline mode requires a filename, so error out if none was provided.
     if args.inline and not args.filename:  # pragma: no cover
@@ -270,23 +323,46 @@ def cli(inline_args: list[str] | None = None) -> int:
         print("Error: You cannot pass both --inline and --output", file=sys.stderr)
         sys.exit(1)
 
-    ensure_ascii = False
-    if args.ensure_ascii:
-        ensure_ascii = True
+    if args.schema and args.schema_model:
+        print("Error: You cannot pass both --schema and --schema-model", file=sys.stderr)
+        sys.exit(1)
+
+    if args.strict and (args.schema or args.schema_model):
+        print("Error: --strict cannot be used with --schema or --schema-model", file=sys.stderr)
+        sys.exit(1)
+
+    ensure_ascii = args.ensure_ascii
 
     try:
+        schema = None
+        if args.schema:
+            with Path(args.schema).open() as fd:
+                schema = json.load(fd)
+        elif args.schema_model:
+            schema = load_schema_model(args.schema_model)
+
         # Use from_file if a filename is provided; otherwise read from stdin.
         if args.filename:
-            result = from_file(args.filename, strict=args.strict)
+            result = from_file(
+                args.filename,
+                skip_json_loads=args.skip_json_loads,
+                strict=args.strict,
+                schema=schema,
+            )
         else:
             data = sys.stdin.read()
-            result = loads(data, strict=args.strict)
+            result = loads(
+                data,
+                skip_json_loads=args.skip_json_loads,
+                strict=args.strict,
+                schema=schema,
+            )
         if args.inline or args.output:
-            with open(args.output or args.filename, mode="w") as fd:
+            with Path(args.output or args.filename).open(mode="w") as fd:
                 json.dump(result, fd, indent=args.indent, ensure_ascii=ensure_ascii)
         else:
             print(json.dumps(result, indent=args.indent, ensure_ascii=ensure_ascii))
-    except Exception as e:  # pragma: no cover
+    except (OSError, TypeError, ValueError) as e:  # pragma: no cover
         print(f"Error: {str(e)}", file=sys.stderr)
         return 1
 

json_repair/parse_array.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from .utils.constants import STRING_DELIMITERS, JSONReturnType
 from .utils.json_context import ContextValues
@@ -6,51 +6,112 @@ from .utils.object_comparer import ObjectComparer
 
 if TYPE_CHECKING:
     from .json_parser import JSONParser
+    from .schema_repair import SchemaRepairer
 
 
-def parse_array(self: "JSONParser") -> list[JSONReturnType]:
+def parse_array(
+    self: "JSONParser",
+    schema: dict[str, Any] | bool | None = None,
+    path: str = "$",
+) -> list[JSONReturnType]:
     # <array> ::= '[' [ <json> *(', ' <json>) ] ']' ; A sequence of JSON values separated by commas
-    arr = []
+    # Only activate schema-guided parsing if a repairer is available and schema looks array-like.
+    schema_repairer: SchemaRepairer | None = None
+    items_schema: object | None = None
+    additional_items: object | None = None
+    if schema is not None and schema is not True:
+        repairer = self.schema_repairer
+        if repairer is not None:
+            schema = repairer.resolve_schema(schema)
+            if schema is False:
+                raise ValueError("Schema does not allow any values.")
+            if schema is not True and repairer.is_array_schema(schema):
+                schema_repairer = repairer
+                items_schema = schema.get("items")
+                additional_items = schema.get("additionalItems", None)
+
+    arr: list[JSONReturnType] = []
     self.context.set(ContextValues.ARRAY)
-    # Stop when you either find the closing parentheses or you have iterated over the entire string
     char = self.get_char_at()
+    idx = 0
+
     while char and char not in ["]", "}"]:
         self.skip_whitespaces()
-        value: JSONReturnType = ""
+
+        # Resolve per-item schema (tuple schemas + additionalItems) when schema guidance is active.
+        item_schema: dict[str, Any] | bool | None = None
+        drop_item = False
+        if schema_repairer is not None:
+            if isinstance(items_schema, list):
+                if idx < len(items_schema):
+                    raw_schema = items_schema[idx]
+                    # Tuple schemas must contain dict/bool entries only.
+                    if raw_schema is not None and not isinstance(raw_schema, (dict, bool)):
+                        raise ValueError("Schema must be an object.")
+                    item_schema = raw_schema
+                else:
+                    if additional_items is False:
+                        drop_item = True
+                    elif isinstance(additional_items, dict):
+                        item_schema = additional_items
+                    else:
+                        item_schema = True
+            elif isinstance(items_schema, dict):
+                item_schema = items_schema
+            else:
+                item_schema = True
+
+        item_path = f"{path}[{idx}]"
+
         if char in STRING_DELIMITERS:
-            # Sometimes it can happen that LLMs forget to start an object and then you think it's a string in an array
-            # So we are going to check if this string is followed by a : or not
-            # And either parse the string or parse the object
+            # A string followed by ':' is often a missing object start; treat it as an object.
             i = 1
             i = self.skip_to_character(char, i)
             i = self.scroll_whitespaces(idx=i + 1)
-            value = self.parse_object() if self.get_char_at(i) == ":" else self.parse_string()
+            if self.get_char_at(i) == ":":
+                if schema_repairer is not None and not drop_item:
+                    # Schema-guided object parsing, then enforce schema on the parsed object.
+                    value = self.parse_object(item_schema, item_path)
+                    value = schema_repairer.repair_value(value, item_schema, item_path)
+                else:
+                    # No schema (or dropping): still parse to keep the cursor in sync.
+                    value = self.parse_object()
+            else:
+                value = self.parse_string()
+                if schema_repairer is not None and not drop_item:
+                    # Apply schema constraints/coercions to scalar values when configured.
+                    value = schema_repairer.repair_value(value, item_schema, item_path)
         else:
-            value = self.parse_json()
+            if schema_repairer is not None and not drop_item:
+                # Use schema-aware parsing to guide nested repairs.
+                value = self.parse_json(item_schema, item_path)
+            else:
+                # Parse normally (or discard) to keep the index aligned.
+                value = self.parse_json()
 
-        # It is possible that parse_json() returns nothing valid, so we increase by 1, unless we find an array separator
         if ObjectComparer.is_strictly_empty(value) and self.get_char_at() not in ["]", ","]:
             self.index += 1
         elif value == "..." and self.get_char_at(-1) == ".":
             self.log(
                 "While parsing an array, found a stray '...'; ignoring it",
             )
-        else:
+        elif not drop_item:
             arr.append(value)
+        elif schema_repairer is not None:
+            # Record drops for visibility when schema forbids extra tuple items.
+            schema_repairer._log("Dropped extra array item not covered by schema", item_path)
 
-        # skip over whitespace after a value but before closing ]
+        idx += 1
         char = self.get_char_at()
         while char and char != "]" and (char.isspace() or char == ","):
             self.index += 1
             char = self.get_char_at()
 
-    # Especially at the end of an LLM generated json you might miss the last "]"
     if char != "]":
         self.log(
             "While parsing an array we missed the closing ], ignoring it",
         )
 
     self.index += 1
-
     self.context.reset()
     return arr

json_repair/parse_comment.py

@@ -67,5 +67,4 @@ def parse_comment(self: "JSONParser") -> JSONReturnType:
             self.index += 1
     if self.context.empty:
         return self.parse_json()
-    else:
-        return ""
+    return ""

json_repair/parse_number.py

@@ -33,7 +33,6 @@ def parse_number(self: "JSONParser") -> JSONReturnType:
             return number_str
         if "." in number_str or "e" in number_str or "E" in number_str:
             return float(number_str)
-        else:
-            return int(number_str)
+        return int(number_str)
     except ValueError:
         return number_str