json-repair 0.55.2__tar.gz → 0.56.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {json_repair-0.55.2 → json_repair-0.56.0}/PKG-INFO +67 -3
- json_repair-0.55.2/src/json_repair.egg-info/PKG-INFO → json_repair-0.56.0/README.md +63 -19
- {json_repair-0.55.2 → json_repair-0.56.0}/pyproject.toml +28 -1
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/json_parser.py +59 -19
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/json_repair.py +91 -12
- json_repair-0.56.0/src/json_repair/parse_array.py +117 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/parse_object.py +107 -10
- json_repair-0.56.0/src/json_repair/schema_repair.py +508 -0
- json_repair-0.56.0/src/json_repair/utils/constants.py +15 -0
- json_repair-0.55.2/README.md → json_repair-0.56.0/src/json_repair.egg-info/PKG-INFO +83 -2
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair.egg-info/SOURCES.txt +5 -0
- json_repair-0.56.0/src/json_repair.egg-info/requires.txt +4 -0
- json_repair-0.56.0/tests/test_repair_json_cli.py +178 -0
- json_repair-0.56.0/tests/test_schema_guided_parse.py +75 -0
- json_repair-0.56.0/tests/test_schema_parser_paths.py +222 -0
- json_repair-0.56.0/tests/test_schema_repairer.py +384 -0
- json_repair-0.55.2/src/json_repair/parse_array.py +0 -56
- json_repair-0.55.2/src/json_repair/utils/constants.py +0 -4
- json_repair-0.55.2/tests/test_repair_json_cli.py +0 -70
- {json_repair-0.55.2 → json_repair-0.56.0}/LICENSE +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/setup.cfg +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/__init__.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/__main__.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/parse_comment.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/parse_number.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/parse_string.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/parse_string_helpers/parse_boolean_or_null.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/parse_string_helpers/parse_json_llm_block.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/py.typed +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/utils/json_context.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/utils/object_comparer.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair/utils/string_file_wrapper.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair.egg-info/dependency_links.txt +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair.egg-info/entry_points.txt +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/src/json_repair.egg-info/top_level.txt +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_json_repair.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_parse_array.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_parse_comment.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_parse_number.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_parse_object.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_parse_string.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_performance.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_repair_json_from_file.py +0 -0
- {json_repair-0.55.2 → json_repair-0.56.0}/tests/test_strict_mode.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: json_repair
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.56.0
|
|
4
4
|
Summary: A package to repair broken json strings
|
|
5
5
|
Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -13,6 +13,9 @@ Classifier: Operating System :: OS Independent
|
|
|
13
13
|
Requires-Python: >=3.10
|
|
14
14
|
Description-Content-Type: text/markdown
|
|
15
15
|
License-File: LICENSE
|
|
16
|
+
Provides-Extra: schema
|
|
17
|
+
Requires-Dist: jsonschema>=4.21; extra == "schema"
|
|
18
|
+
Requires-Dist: pydantic>=2; extra == "schema"
|
|
16
19
|
Dynamic: license-file
|
|
17
20
|
|
|
18
21
|
[](https://pypi.org/project/json-repair/)
|
|
@@ -190,6 +193,58 @@ In strict mode the parser raises `ValueError` as soon as it encounters structura
|
|
|
190
193
|
|
|
191
194
|
Strict mode still honors `skip_json_loads=True`; combining them lets you skip the initial `json.loads` check but still enforce strict parsing rules.
|
|
192
195
|
|
|
196
|
+
### Schema-guided repairs
|
|
197
|
+
|
|
198
|
+
**Alpha feature (not yet in stable releases).** Schema-guided repairs are currently shipped only in alpha builds (e.g., `0.56.0-alpha.*`). The API and behavior may change or break between alpha releases.
|
|
199
|
+
|
|
200
|
+
You can guide repairs with a JSON Schema (or a Pydantic v2 model). When enabled, the parser will:
|
|
201
|
+
|
|
202
|
+
- Fill missing values (defaults, required fields).
|
|
203
|
+
- Coerce scalars where safe (e.g., `"1"` → `1` for integer fields).
|
|
204
|
+
- Drop properties/items that the schema disallows.
|
|
205
|
+
|
|
206
|
+
This is especially useful when you need deterministic, schema-valid outputs for downstream validation, storage, or typed processing. If the input cannot be repaired to satisfy the schema, `json_repair` raises `ValueError`.
|
|
207
|
+
|
|
208
|
+
Install the optional dependencies:
|
|
209
|
+
|
|
210
|
+
pip install 'json-repair[schema]'
|
|
211
|
+
|
|
212
|
+
(For CLI usage, you can also use `pipx install 'json-repair[schema]'`.)
|
|
213
|
+
|
|
214
|
+
Schema guidance is skipped for already-valid JSON unless you pass `skip_json_loads=True` (this forces the parser to run even on valid JSON). Schema guidance is mutually exclusive with `strict=True`.
|
|
215
|
+
|
|
216
|
+
```
|
|
217
|
+
from json_repair import repair_json
|
|
218
|
+
|
|
219
|
+
schema = {
|
|
220
|
+
"type": "object",
|
|
221
|
+
"properties": {"value": {"type": "integer"}},
|
|
222
|
+
"required": ["value"],
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
repair_json('{"value": "1"}', schema=schema, skip_json_loads=True, return_objects=True)
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
Pydantic v2 model example:
|
|
229
|
+
|
|
230
|
+
```
|
|
231
|
+
from pydantic import BaseModel, Field
|
|
232
|
+
from json_repair import repair_json
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class Payload(BaseModel):
|
|
236
|
+
value: int
|
|
237
|
+
tags: list[str] = Field(default_factory=list)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
repair_json(
|
|
241
|
+
'{"value": "1", "tags": }',
|
|
242
|
+
schema=Payload,
|
|
243
|
+
skip_json_loads=True,
|
|
244
|
+
return_objects=True,
|
|
245
|
+
)
|
|
246
|
+
```
|
|
247
|
+
|
|
193
248
|
### Use json_repair with streaming
|
|
194
249
|
|
|
195
250
|
Sometimes you are streaming some data and want to repair the JSON coming from it. Normally this won't work but you can pass `stream_stable` to `repair_json()` or `loads()` to make it work:
|
|
@@ -207,7 +262,9 @@ pipx install json-repair
|
|
|
207
262
|
to know all options available:
|
|
208
263
|
```
|
|
209
264
|
$ json_repair -h
|
|
210
|
-
usage: json_repair [-h] [-i] [-o TARGET] [--ensure_ascii] [--indent INDENT]
|
|
265
|
+
usage: json_repair [-h] [-i] [-o TARGET] [--ensure_ascii] [--indent INDENT]
|
|
266
|
+
[--skip-json-loads] [--schema SCHEMA] [--schema-model MODEL]
|
|
267
|
+
[--strict] [filename]
|
|
211
268
|
|
|
212
269
|
Repair and parse JSON files.
|
|
213
270
|
|
|
@@ -221,6 +278,9 @@ options:
|
|
|
221
278
|
If specified, the output will be written to TARGET filename instead of stdout
|
|
222
279
|
--ensure_ascii Pass ensure_ascii=True to json.dumps()
|
|
223
280
|
--indent INDENT Number of spaces for indentation (Default 2)
|
|
281
|
+
--skip-json-loads Skip initial json.loads validation (needed to force schema on valid JSON)
|
|
282
|
+
--schema SCHEMA Path to a JSON Schema file that guides repairs
|
|
283
|
+
--schema-model MODEL Pydantic v2 model in 'module:ClassName' form that guides repairs
|
|
224
284
|
--strict Raise on duplicate keys, missing separators, empty keys/values, and similar structural issues instead of repairing them
|
|
225
285
|
```
|
|
226
286
|
|
|
@@ -278,7 +338,11 @@ I am sure some corner cases will be missing, if you have examples please open an
|
|
|
278
338
|
If you want to contribute, start with `CONTRIBUTING.md` and read the Code Wiki writeup for a tour of the codebase and key entry points: https://codewiki.google/github.com/mangiucugna/json_repair
|
|
279
339
|
|
|
280
340
|
# How to develop
|
|
281
|
-
|
|
341
|
+
Use `uv` to set up the dev environment and run tooling:
|
|
342
|
+
|
|
343
|
+
uv sync --group dev
|
|
344
|
+
uv run pre-commit run --all-files
|
|
345
|
+
uv run pytest
|
|
282
346
|
|
|
283
347
|
Make sure that the Github Actions running after pushing a new commit don't fail as well.
|
|
284
348
|
|
|
@@ -1,20 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: json_repair
|
|
3
|
-
Version: 0.55.2
|
|
4
|
-
Summary: A package to repair broken json strings
|
|
5
|
-
Author-email: Stefano Baccianella <4247706+mangiucugna@users.noreply.github.com>
|
|
6
|
-
License-Expression: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/mangiucugna/json_repair/
|
|
8
|
-
Project-URL: Bug Tracker, https://github.com/mangiucugna/json_repair/issues
|
|
9
|
-
Project-URL: Live demo, https://mangiucugna.github.io/json_repair/
|
|
10
|
-
Keywords: JSON,REPAIR,LLM,PARSER
|
|
11
|
-
Classifier: Programming Language :: Python :: 3
|
|
12
|
-
Classifier: Operating System :: OS Independent
|
|
13
|
-
Requires-Python: >=3.10
|
|
14
|
-
Description-Content-Type: text/markdown
|
|
15
|
-
License-File: LICENSE
|
|
16
|
-
Dynamic: license-file
|
|
17
|
-
|
|
18
1
|
[](https://pypi.org/project/json-repair/)
|
|
19
2
|

|
|
20
3
|
[](https://pypi.org/project/json-repair/)
|
|
@@ -190,6 +173,58 @@ In strict mode the parser raises `ValueError` as soon as it encounters structura
|
|
|
190
173
|
|
|
191
174
|
Strict mode still honors `skip_json_loads=True`; combining them lets you skip the initial `json.loads` check but still enforce strict parsing rules.
|
|
192
175
|
|
|
176
|
+
### Schema-guided repairs
|
|
177
|
+
|
|
178
|
+
**Alpha feature (not yet in stable releases).** Schema-guided repairs are currently shipped only in alpha builds (e.g., `0.56.0-alpha.*`). The API and behavior may change or break between alpha releases.
|
|
179
|
+
|
|
180
|
+
You can guide repairs with a JSON Schema (or a Pydantic v2 model). When enabled, the parser will:
|
|
181
|
+
|
|
182
|
+
- Fill missing values (defaults, required fields).
|
|
183
|
+
- Coerce scalars where safe (e.g., `"1"` → `1` for integer fields).
|
|
184
|
+
- Drop properties/items that the schema disallows.
|
|
185
|
+
|
|
186
|
+
This is especially useful when you need deterministic, schema-valid outputs for downstream validation, storage, or typed processing. If the input cannot be repaired to satisfy the schema, `json_repair` raises `ValueError`.
|
|
187
|
+
|
|
188
|
+
Install the optional dependencies:
|
|
189
|
+
|
|
190
|
+
pip install 'json-repair[schema]'
|
|
191
|
+
|
|
192
|
+
(For CLI usage, you can also use `pipx install 'json-repair[schema]'`.)
|
|
193
|
+
|
|
194
|
+
Schema guidance is skipped for already-valid JSON unless you pass `skip_json_loads=True` (this forces the parser to run even on valid JSON). Schema guidance is mutually exclusive with `strict=True`.
|
|
195
|
+
|
|
196
|
+
```
|
|
197
|
+
from json_repair import repair_json
|
|
198
|
+
|
|
199
|
+
schema = {
|
|
200
|
+
"type": "object",
|
|
201
|
+
"properties": {"value": {"type": "integer"}},
|
|
202
|
+
"required": ["value"],
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
repair_json('{"value": "1"}', schema=schema, skip_json_loads=True, return_objects=True)
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Pydantic v2 model example:
|
|
209
|
+
|
|
210
|
+
```
|
|
211
|
+
from pydantic import BaseModel, Field
|
|
212
|
+
from json_repair import repair_json
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class Payload(BaseModel):
|
|
216
|
+
value: int
|
|
217
|
+
tags: list[str] = Field(default_factory=list)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
repair_json(
|
|
221
|
+
'{"value": "1", "tags": }',
|
|
222
|
+
schema=Payload,
|
|
223
|
+
skip_json_loads=True,
|
|
224
|
+
return_objects=True,
|
|
225
|
+
)
|
|
226
|
+
```
|
|
227
|
+
|
|
193
228
|
### Use json_repair with streaming
|
|
194
229
|
|
|
195
230
|
Sometimes you are streaming some data and want to repair the JSON coming from it. Normally this won't work but you can pass `stream_stable` to `repair_json()` or `loads()` to make it work:
|
|
@@ -207,7 +242,9 @@ pipx install json-repair
|
|
|
207
242
|
to know all options available:
|
|
208
243
|
```
|
|
209
244
|
$ json_repair -h
|
|
210
|
-
usage: json_repair [-h] [-i] [-o TARGET] [--ensure_ascii] [--indent INDENT]
|
|
245
|
+
usage: json_repair [-h] [-i] [-o TARGET] [--ensure_ascii] [--indent INDENT]
|
|
246
|
+
[--skip-json-loads] [--schema SCHEMA] [--schema-model MODEL]
|
|
247
|
+
[--strict] [filename]
|
|
211
248
|
|
|
212
249
|
Repair and parse JSON files.
|
|
213
250
|
|
|
@@ -221,6 +258,9 @@ options:
|
|
|
221
258
|
If specified, the output will be written to TARGET filename instead of stdout
|
|
222
259
|
--ensure_ascii Pass ensure_ascii=True to json.dumps()
|
|
223
260
|
--indent INDENT Number of spaces for indentation (Default 2)
|
|
261
|
+
--skip-json-loads Skip initial json.loads validation (needed to force schema on valid JSON)
|
|
262
|
+
--schema SCHEMA Path to a JSON Schema file that guides repairs
|
|
263
|
+
--schema-model MODEL Pydantic v2 model in 'module:ClassName' form that guides repairs
|
|
224
264
|
--strict Raise on duplicate keys, missing separators, empty keys/values, and similar structural issues instead of repairing them
|
|
225
265
|
```
|
|
226
266
|
|
|
@@ -278,7 +318,11 @@ I am sure some corner cases will be missing, if you have examples please open an
|
|
|
278
318
|
If you want to contribute, start with `CONTRIBUTING.md` and read the Code Wiki writeup for a tour of the codebase and key entry points: https://codewiki.google/github.com/mangiucugna/json_repair
|
|
279
319
|
|
|
280
320
|
# How to develop
|
|
281
|
-
|
|
321
|
+
Use `uv` to set up the dev environment and run tooling:
|
|
322
|
+
|
|
323
|
+
uv sync --group dev
|
|
324
|
+
uv run pre-commit run --all-files
|
|
325
|
+
uv run pytest
|
|
282
326
|
|
|
283
327
|
Make sure that the Github Actions running after pushing a new commit don't fail as well.
|
|
284
328
|
|
|
@@ -3,7 +3,7 @@ requires = ["setuptools>=61.0"]
|
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
[project]
|
|
5
5
|
name = "json_repair"
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.56.0"
|
|
7
7
|
license = "MIT"
|
|
8
8
|
license-files = ["LICENSE"]
|
|
9
9
|
authors = [
|
|
@@ -21,6 +21,33 @@ classifiers = [
|
|
|
21
21
|
"Homepage" = "https://github.com/mangiucugna/json_repair/"
|
|
22
22
|
"Bug Tracker" = "https://github.com/mangiucugna/json_repair/issues"
|
|
23
23
|
"Live demo" = "https://mangiucugna.github.io/json_repair/"
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
schema = [
|
|
26
|
+
"jsonschema>=4.21",
|
|
27
|
+
"pydantic>=2",
|
|
28
|
+
]
|
|
29
|
+
[dependency-groups]
|
|
30
|
+
dev = [
|
|
31
|
+
"coverage",
|
|
32
|
+
"jsonschema",
|
|
33
|
+
"mypy",
|
|
34
|
+
"pydantic",
|
|
35
|
+
"pre-commit",
|
|
36
|
+
"pytest",
|
|
37
|
+
"pytest-benchmark",
|
|
38
|
+
"ty",
|
|
39
|
+
]
|
|
40
|
+
test = [
|
|
41
|
+
"coverage",
|
|
42
|
+
"jsonschema",
|
|
43
|
+
"pydantic",
|
|
44
|
+
"pytest",
|
|
45
|
+
"pytest-benchmark",
|
|
46
|
+
]
|
|
47
|
+
typecheck = [
|
|
48
|
+
"mypy",
|
|
49
|
+
"ty",
|
|
50
|
+
]
|
|
24
51
|
[tool.pytest.ini_options]
|
|
25
52
|
pythonpath = [
|
|
26
53
|
"."
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
from
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from typing import TYPE_CHECKING, Any, TextIO
|
|
2
3
|
|
|
3
4
|
from .parse_array import parse_array as _parse_array
|
|
4
5
|
from .parse_comment import parse_comment as _parse_comment
|
|
@@ -10,11 +11,18 @@ from .utils.json_context import JsonContext
|
|
|
10
11
|
from .utils.object_comparer import ObjectComparer
|
|
11
12
|
from .utils.string_file_wrapper import StringFileWrapper
|
|
12
13
|
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from .schema_repair import SchemaRepairer
|
|
16
|
+
|
|
13
17
|
|
|
14
18
|
class JSONParser:
|
|
15
19
|
# Split the parse methods into separate files because this one was like 3000 lines
|
|
16
|
-
def parse_array(
|
|
17
|
-
|
|
20
|
+
def parse_array(
|
|
21
|
+
self,
|
|
22
|
+
schema: dict[str, Any] | bool | None = None,
|
|
23
|
+
path: str = "$",
|
|
24
|
+
) -> list[JSONReturnType]:
|
|
25
|
+
return _parse_array(self, schema, path)
|
|
18
26
|
|
|
19
27
|
def parse_comment(self) -> JSONReturnType:
|
|
20
28
|
return _parse_comment(self)
|
|
@@ -22,8 +30,12 @@ class JSONParser:
|
|
|
22
30
|
def parse_number(self) -> JSONReturnType:
|
|
23
31
|
return _parse_number(self)
|
|
24
32
|
|
|
25
|
-
def parse_object(
|
|
26
|
-
|
|
33
|
+
def parse_object(
|
|
34
|
+
self,
|
|
35
|
+
schema: dict[str, Any] | bool | None = None,
|
|
36
|
+
path: str = "$",
|
|
37
|
+
) -> JSONReturnType:
|
|
38
|
+
return _parse_object(self, schema, path)
|
|
27
39
|
|
|
28
40
|
def parse_string(self) -> JSONReturnType:
|
|
29
41
|
return _parse_string(self)
|
|
@@ -53,8 +65,8 @@ class JSONParser:
|
|
|
53
65
|
# We could add a guard in the code for each call but that would make this code unreadable, so here's this neat trick
|
|
54
66
|
# Replace self.log with a noop
|
|
55
67
|
self.logging = logging
|
|
68
|
+
self.logger: list[dict[str, str]] = []
|
|
56
69
|
if logging:
|
|
57
|
-
self.logger: list[dict[str, str]] = []
|
|
58
70
|
self.log = self._log
|
|
59
71
|
else:
|
|
60
72
|
# No-op
|
|
@@ -71,11 +83,26 @@ class JSONParser:
|
|
|
71
83
|
# may not be desirable in some use cases and the user would prefer json_repair to return an exception.
|
|
72
84
|
# So strict mode was added to disable some of those heuristics.
|
|
73
85
|
self.strict = strict
|
|
86
|
+
self.schema_repairer: SchemaRepairer | None = None
|
|
74
87
|
|
|
75
88
|
def parse(
|
|
76
89
|
self,
|
|
77
|
-
) -> JSONReturnType
|
|
78
|
-
|
|
90
|
+
) -> JSONReturnType:
|
|
91
|
+
return self._parse_top_level(self.parse_json)
|
|
92
|
+
|
|
93
|
+
def parse_with_schema(
|
|
94
|
+
self,
|
|
95
|
+
repairer: "SchemaRepairer",
|
|
96
|
+
schema: dict[str, Any] | bool,
|
|
97
|
+
) -> JSONReturnType:
|
|
98
|
+
"""Parse with schema guidance enabled for all nested values."""
|
|
99
|
+
self.schema_repairer = repairer
|
|
100
|
+
return self._parse_top_level(lambda: self.parse_json(schema, "$"))
|
|
101
|
+
|
|
102
|
+
# Consolidate top-level parsing so we handle multiple sequential JSON values consistently
|
|
103
|
+
# (including update semantics and strict-mode validation).
|
|
104
|
+
def _parse_top_level(self, parse_element: Callable[[], JSONReturnType]) -> JSONReturnType:
|
|
105
|
+
json = parse_element()
|
|
79
106
|
if self.index < len(self.json_str):
|
|
80
107
|
self.log(
|
|
81
108
|
"The parser returned early, checking if there's more json elements",
|
|
@@ -83,19 +110,17 @@ class JSONParser:
|
|
|
83
110
|
json = [json]
|
|
84
111
|
while self.index < len(self.json_str):
|
|
85
112
|
self.context.reset()
|
|
86
|
-
j =
|
|
113
|
+
j = parse_element()
|
|
87
114
|
if j:
|
|
88
115
|
if ObjectComparer.is_same_object(json[-1], j):
|
|
89
|
-
#
|
|
116
|
+
# Treat repeated objects as updates: keep the newest value.
|
|
90
117
|
json.pop()
|
|
91
118
|
else:
|
|
92
119
|
if not json[-1]:
|
|
93
120
|
json.pop()
|
|
94
121
|
json.append(j)
|
|
95
122
|
else:
|
|
96
|
-
# this was a bust, move the index
|
|
97
123
|
self.index += 1
|
|
98
|
-
# If nothing extra was found, don't return an array
|
|
99
124
|
if len(json) == 1:
|
|
100
125
|
self.log(
|
|
101
126
|
"There were no more elements, returning the element without the array",
|
|
@@ -106,13 +131,23 @@ class JSONParser:
|
|
|
106
131
|
"Multiple top-level JSON elements found in strict mode, raising an error",
|
|
107
132
|
)
|
|
108
133
|
raise ValueError("Multiple top-level JSON elements found in strict mode.")
|
|
109
|
-
if self.logging:
|
|
110
|
-
return json, self.logger
|
|
111
134
|
return json
|
|
112
135
|
|
|
113
136
|
def parse_json(
|
|
114
137
|
self,
|
|
138
|
+
schema: dict[str, Any] | bool | None = None,
|
|
139
|
+
path: str = "$",
|
|
115
140
|
) -> JSONReturnType:
|
|
141
|
+
"""Parse the next JSON value and, when configured, enforce schema constraints."""
|
|
142
|
+
repairer = self.schema_repairer if self.schema_repairer is not None and schema not in (None, True) else None
|
|
143
|
+
if repairer is not None:
|
|
144
|
+
# Resolve references once and decide whether schema-guided repairs are needed.
|
|
145
|
+
schema = repairer.resolve_schema(schema)
|
|
146
|
+
if schema is True:
|
|
147
|
+
repairer = None
|
|
148
|
+
elif schema is False:
|
|
149
|
+
raise ValueError("Schema does not allow any values.")
|
|
150
|
+
|
|
116
151
|
while True:
|
|
117
152
|
char = self.get_char_at()
|
|
118
153
|
# None means that we are at the end of the string provided
|
|
@@ -121,19 +156,24 @@ class JSONParser:
|
|
|
121
156
|
# <object> starts with '{'
|
|
122
157
|
if char == "{":
|
|
123
158
|
self.index += 1
|
|
124
|
-
|
|
159
|
+
value = self.parse_object(schema, path) if repairer else self.parse_object()
|
|
160
|
+
return repairer.repair_value(value, schema, path) if repairer else value
|
|
125
161
|
# <array> starts with '['
|
|
126
162
|
if char == "[":
|
|
127
163
|
self.index += 1
|
|
128
|
-
|
|
164
|
+
value = self.parse_array(schema, path) if repairer else self.parse_array()
|
|
165
|
+
return repairer.repair_value(value, schema, path) if repairer else value
|
|
129
166
|
# <string> starts with a quote
|
|
130
167
|
if not self.context.empty and (char in STRING_DELIMITERS or char.isalpha()):
|
|
131
|
-
|
|
168
|
+
value = self.parse_string()
|
|
169
|
+
return repairer.repair_value(value, schema, path) if repairer else value
|
|
132
170
|
# <number> starts with [0-9] or minus
|
|
133
171
|
if not self.context.empty and (char.isdigit() or char == "-" or char == "."):
|
|
134
|
-
|
|
172
|
+
value = self.parse_number()
|
|
173
|
+
return repairer.repair_value(value, schema, path) if repairer else value
|
|
135
174
|
if char in ["#", "/"]:
|
|
136
|
-
|
|
175
|
+
value = self.parse_comment()
|
|
176
|
+
return repairer.repair_value(value, schema, path) if repairer else value
|
|
137
177
|
# If everything else fails, we just ignore and move on
|
|
138
178
|
self.index += 1
|
|
139
179
|
|
|
@@ -29,6 +29,7 @@ from pathlib import Path
|
|
|
29
29
|
from typing import Any, Literal, TextIO, overload
|
|
30
30
|
|
|
31
31
|
from .json_parser import JSONParser
|
|
32
|
+
from .schema_repair import SchemaRepairer, load_schema_model, schema_from_input
|
|
32
33
|
from .utils.constants import JSONReturnType
|
|
33
34
|
|
|
34
35
|
|
|
@@ -42,6 +43,7 @@ def repair_json(
|
|
|
42
43
|
chunk_length: int = 0,
|
|
43
44
|
stream_stable: bool = False,
|
|
44
45
|
strict: bool = False,
|
|
46
|
+
schema: Any | None = None,
|
|
45
47
|
**json_dumps_args: Any,
|
|
46
48
|
) -> str: ...
|
|
47
49
|
|
|
@@ -56,6 +58,7 @@ def repair_json(
|
|
|
56
58
|
chunk_length: int = 0,
|
|
57
59
|
stream_stable: bool = False,
|
|
58
60
|
strict: bool = False,
|
|
61
|
+
schema: Any | None = None,
|
|
59
62
|
**json_dumps_args: Any,
|
|
60
63
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]: ...
|
|
61
64
|
|
|
@@ -69,6 +72,7 @@ def repair_json(
|
|
|
69
72
|
chunk_length: int = 0,
|
|
70
73
|
stream_stable: bool = False,
|
|
71
74
|
strict: bool = False,
|
|
75
|
+
schema: Any | None = None,
|
|
72
76
|
**json_dumps_args: Any,
|
|
73
77
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
|
74
78
|
"""
|
|
@@ -84,24 +88,46 @@ def repair_json(
|
|
|
84
88
|
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Ignored if `json_fd` is None. Do not use! Use `from_file` or `load` instead. Defaults to 1MB.
|
|
85
89
|
stream_stable (bool, optional): When the json to be repaired is the accumulation of streaming json at a certain moment.If this parameter to True will keep the repair results stable.
|
|
86
90
|
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
|
91
|
+
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs. Schema guidance is skipped for already-valid JSON unless `skip_json_loads=True`.
|
|
87
92
|
Returns:
|
|
88
93
|
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON or a tuple with the repaired JSON and repair log when logging is True.
|
|
89
94
|
"""
|
|
95
|
+
# Schema-guided repairs and strict mode are mutually exclusive to avoid conflicting behavior.
|
|
96
|
+
if schema is not None and strict:
|
|
97
|
+
raise ValueError("schema and strict cannot be used together.")
|
|
98
|
+
|
|
90
99
|
parser = JSONParser(json_str, json_fd, logging, chunk_length, stream_stable, strict)
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
100
|
+
# When JSON is already valid, skip schema guidance unless the caller explicitly disables json.loads.
|
|
101
|
+
if not skip_json_loads:
|
|
102
|
+
loaded_json: JSONReturnType | None
|
|
94
103
|
try:
|
|
95
|
-
|
|
104
|
+
loaded_json = json.load(json_fd) if json_fd else json.loads(json_str)
|
|
96
105
|
except json.JSONDecodeError:
|
|
97
|
-
|
|
106
|
+
loaded_json = None
|
|
107
|
+
else:
|
|
108
|
+
if logging:
|
|
109
|
+
return loaded_json, []
|
|
110
|
+
if return_objects:
|
|
111
|
+
return loaded_json
|
|
112
|
+
if loaded_json == "":
|
|
113
|
+
return ""
|
|
114
|
+
return json.dumps(loaded_json, **json_dumps_args)
|
|
115
|
+
|
|
116
|
+
# Schema guidance only happens in parser mode.
|
|
117
|
+
schema_obj = schema_from_input(schema) if schema is not None else None
|
|
118
|
+
parsed_json: JSONReturnType
|
|
119
|
+
if schema_obj is None:
|
|
120
|
+
parsed_json = parser.parse()
|
|
121
|
+
else:
|
|
122
|
+
repairer = SchemaRepairer(schema_obj, parser.logger if logging else None)
|
|
123
|
+
parsed_json = parser.parse_with_schema(repairer, schema_obj)
|
|
124
|
+
# Post-parse validation ensures we reject values that cannot satisfy the schema.
|
|
125
|
+
repairer.validate(parsed_json, schema_obj)
|
|
98
126
|
# It's useful to return the actual object instead of the json string,
|
|
99
127
|
# it allows this lib to be a replacement of the json library
|
|
100
|
-
if
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if logging and not isinstance(parsed_json, tuple):
|
|
104
|
-
return parsed_json, []
|
|
128
|
+
if logging:
|
|
129
|
+
return parsed_json, parser.logger
|
|
130
|
+
if return_objects:
|
|
105
131
|
return parsed_json
|
|
106
132
|
# Avoid returning only a pair of quotes if it's an empty string
|
|
107
133
|
if parsed_json == "":
|
|
@@ -115,6 +141,7 @@ def loads(
|
|
|
115
141
|
logging: bool = False,
|
|
116
142
|
stream_stable: bool = False,
|
|
117
143
|
strict: bool = False,
|
|
144
|
+
schema: Any | None = None,
|
|
118
145
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]] | str:
|
|
119
146
|
"""
|
|
120
147
|
This function works like `json.loads()` except that it will fix your JSON in the process.
|
|
@@ -125,6 +152,7 @@ def loads(
|
|
|
125
152
|
skip_json_loads (bool, optional): If True, skip calling the built-in json.loads() function to verify that the json is valid before attempting to repair. Defaults to False.
|
|
126
153
|
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
|
127
154
|
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
|
155
|
+
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs. Schema guidance is skipped for already-valid JSON unless `skip_json_loads=True`.
|
|
128
156
|
|
|
129
157
|
Returns:
|
|
130
158
|
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]], str]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
|
@@ -136,6 +164,7 @@ def loads(
|
|
|
136
164
|
logging=logging,
|
|
137
165
|
stream_stable=stream_stable,
|
|
138
166
|
strict=strict,
|
|
167
|
+
schema=schema,
|
|
139
168
|
)
|
|
140
169
|
|
|
141
170
|
|
|
@@ -145,6 +174,7 @@ def load(
|
|
|
145
174
|
logging: bool = False,
|
|
146
175
|
chunk_length: int = 0,
|
|
147
176
|
strict: bool = False,
|
|
177
|
+
schema: Any | None = None,
|
|
148
178
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
|
149
179
|
"""
|
|
150
180
|
This function works like `json.load()` except that it will fix your JSON in the process.
|
|
@@ -156,6 +186,7 @@ def load(
|
|
|
156
186
|
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
|
157
187
|
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
|
|
158
188
|
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
|
189
|
+
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs. Schema guidance is skipped for already-valid JSON unless `skip_json_loads=True`.
|
|
159
190
|
|
|
160
191
|
Returns:
|
|
161
192
|
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
|
@@ -167,6 +198,7 @@ def load(
|
|
|
167
198
|
skip_json_loads=skip_json_loads,
|
|
168
199
|
logging=logging,
|
|
169
200
|
strict=strict,
|
|
201
|
+
schema=schema,
|
|
170
202
|
)
|
|
171
203
|
|
|
172
204
|
|
|
@@ -176,6 +208,7 @@ def from_file(
|
|
|
176
208
|
logging: bool = False,
|
|
177
209
|
chunk_length: int = 0,
|
|
178
210
|
strict: bool = False,
|
|
211
|
+
schema: Any | None = None,
|
|
179
212
|
) -> JSONReturnType | tuple[JSONReturnType, list[dict[str, str]]]:
|
|
180
213
|
"""
|
|
181
214
|
This function is a wrapper around `load()` so you can pass the filename as string
|
|
@@ -186,6 +219,7 @@ def from_file(
|
|
|
186
219
|
logging (bool, optional): If True, return a tuple with the repaired json and a log of all repair actions. Defaults to False.
|
|
187
220
|
chunk_length (int, optional): Size in bytes of the file chunks to read at once. Defaults to 1MB.
|
|
188
221
|
strict (bool, optional): If True, surface structural problems (duplicate keys, missing separators, empty keys/values, etc.) as ValueError instead of repairing them.
|
|
222
|
+
schema (Any, optional): JSON Schema dict, boolean schema, or pydantic v2 model used to guide repairs. Schema guidance is skipped for already-valid JSON unless `skip_json_loads=True`.
|
|
189
223
|
|
|
190
224
|
Returns:
|
|
191
225
|
Union[JSONReturnType, Tuple[JSONReturnType, List[Dict[str, str]]]]: The repaired JSON object or a tuple with the repaired JSON object and repair log.
|
|
@@ -197,6 +231,7 @@ def from_file(
|
|
|
197
231
|
logging=logging,
|
|
198
232
|
chunk_length=chunk_length,
|
|
199
233
|
strict=strict,
|
|
234
|
+
schema=schema,
|
|
200
235
|
)
|
|
201
236
|
|
|
202
237
|
|
|
@@ -211,6 +246,10 @@ def cli(inline_args: list[str] | None = None) -> int:
|
|
|
211
246
|
- -o, --output TARGET (str): If specified, the output will be written to TARGET filename instead of stdout.
|
|
212
247
|
- --ensure_ascii (bool): Pass ensure_ascii=True to json.dumps(). Will pass False otherwise.
|
|
213
248
|
- --indent INDENT (int): Number of spaces for indentation (Default 2).
|
|
249
|
+
- --skip-json-loads (bool): Skip initial json.loads validation (needed to force schema on valid JSON).
|
|
250
|
+
- --schema SCHEMA (str): Path to a JSON Schema file that guides repairs.
|
|
251
|
+
- --schema-model MODEL (str): Pydantic v2 model in 'module:ClassName' form that guides repairs.
|
|
252
|
+
- --strict (bool): Raise on duplicate keys, missing separators, empty keys/values, and other unrecoverable structures instead of repairing them.
|
|
214
253
|
|
|
215
254
|
Returns:
|
|
216
255
|
int: Exit code of the CLI operation.
|
|
@@ -252,6 +291,21 @@ def cli(inline_args: list[str] | None = None) -> int:
|
|
|
252
291
|
default=2,
|
|
253
292
|
help="Number of spaces for indentation (Default 2)",
|
|
254
293
|
)
|
|
294
|
+
parser.add_argument(
|
|
295
|
+
"--skip-json-loads",
|
|
296
|
+
action="store_true",
|
|
297
|
+
help="Skip initial json.loads validation (needed to force schema on valid JSON)",
|
|
298
|
+
)
|
|
299
|
+
parser.add_argument(
|
|
300
|
+
"--schema",
|
|
301
|
+
metavar="SCHEMA",
|
|
302
|
+
help="Path to a JSON Schema file that guides repairs",
|
|
303
|
+
)
|
|
304
|
+
parser.add_argument(
|
|
305
|
+
"--schema-model",
|
|
306
|
+
metavar="MODEL",
|
|
307
|
+
help="Pydantic v2 model in 'module:ClassName' form that guides repairs",
|
|
308
|
+
)
|
|
255
309
|
parser.add_argument(
|
|
256
310
|
"--strict",
|
|
257
311
|
action="store_true",
|
|
@@ -269,15 +323,40 @@ def cli(inline_args: list[str] | None = None) -> int:
|
|
|
269
323
|
print("Error: You cannot pass both --inline and --output", file=sys.stderr)
|
|
270
324
|
sys.exit(1)
|
|
271
325
|
|
|
326
|
+
if args.schema and args.schema_model:
|
|
327
|
+
print("Error: You cannot pass both --schema and --schema-model", file=sys.stderr)
|
|
328
|
+
sys.exit(1)
|
|
329
|
+
|
|
330
|
+
if args.strict and (args.schema or args.schema_model):
|
|
331
|
+
print("Error: --strict cannot be used with --schema or --schema-model", file=sys.stderr)
|
|
332
|
+
sys.exit(1)
|
|
333
|
+
|
|
272
334
|
ensure_ascii = args.ensure_ascii
|
|
273
335
|
|
|
274
336
|
try:
|
|
337
|
+
schema = None
|
|
338
|
+
if args.schema:
|
|
339
|
+
with Path(args.schema).open() as fd:
|
|
340
|
+
schema = json.load(fd)
|
|
341
|
+
elif args.schema_model:
|
|
342
|
+
schema = load_schema_model(args.schema_model)
|
|
343
|
+
|
|
275
344
|
# Use from_file if a filename is provided; otherwise read from stdin.
|
|
276
345
|
if args.filename:
|
|
277
|
-
result = from_file(
|
|
346
|
+
result = from_file(
|
|
347
|
+
args.filename,
|
|
348
|
+
skip_json_loads=args.skip_json_loads,
|
|
349
|
+
strict=args.strict,
|
|
350
|
+
schema=schema,
|
|
351
|
+
)
|
|
278
352
|
else:
|
|
279
353
|
data = sys.stdin.read()
|
|
280
|
-
result = loads(
|
|
354
|
+
result = loads(
|
|
355
|
+
data,
|
|
356
|
+
skip_json_loads=args.skip_json_loads,
|
|
357
|
+
strict=args.strict,
|
|
358
|
+
schema=schema,
|
|
359
|
+
)
|
|
281
360
|
if args.inline or args.output:
|
|
282
361
|
with Path(args.output or args.filename).open(mode="w") as fd:
|
|
283
362
|
json.dump(result, fd, indent=args.indent, ensure_ascii=ensure_ascii)
|