omdev 0.0.0.dev133__py3-none-any.whl → 0.0.0.dev137__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of omdev might be problematic. Click here for more details.
- omdev/.manifests.json +15 -0
- omdev/json/__init__.py +0 -0
- omdev/json/__main__.py +11 -0
- omdev/json/cli.py +299 -0
- omdev/json/formats.py +71 -0
- omdev/json/io.py +74 -0
- omdev/json/parsing.py +83 -0
- omdev/json/processing.py +48 -0
- omdev/json/rendering.py +92 -0
- omdev/packaging/marshal.py +1 -1
- omdev/scripts/pyproject.py +9 -6
- {omdev-0.0.0.dev133.dist-info → omdev-0.0.0.dev137.dist-info}/METADATA +2 -2
- {omdev-0.0.0.dev133.dist-info → omdev-0.0.0.dev137.dist-info}/RECORD +17 -9
- {omdev-0.0.0.dev133.dist-info → omdev-0.0.0.dev137.dist-info}/LICENSE +0 -0
- {omdev-0.0.0.dev133.dist-info → omdev-0.0.0.dev137.dist-info}/WHEEL +0 -0
- {omdev-0.0.0.dev133.dist-info → omdev-0.0.0.dev137.dist-info}/entry_points.txt +0 -0
- {omdev-0.0.0.dev133.dist-info → omdev-0.0.0.dev137.dist-info}/top_level.txt +0 -0
omdev/.manifests.json
CHANGED
|
@@ -95,6 +95,21 @@
|
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
97
|
},
|
|
98
|
+
{
|
|
99
|
+
"module": ".json.__main__",
|
|
100
|
+
"attr": "_CLI_MODULE",
|
|
101
|
+
"file": "omdev/json/__main__.py",
|
|
102
|
+
"line": 4,
|
|
103
|
+
"value": {
|
|
104
|
+
"$.cli.types.CliModule": {
|
|
105
|
+
"cmd_name": [
|
|
106
|
+
"json",
|
|
107
|
+
"j"
|
|
108
|
+
],
|
|
109
|
+
"mod_name": "omdev.json.__main__"
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
},
|
|
98
113
|
{
|
|
99
114
|
"module": ".magic.find",
|
|
100
115
|
"attr": "_CLI_MODULE",
|
omdev/json/__init__.py
ADDED
|
File without changes
|
omdev/json/__main__.py
ADDED
omdev/json/cli.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TODO:
|
|
3
|
+
- read from http
|
|
4
|
+
- jmespath output flat, unquoted strs like jq '.[]'
|
|
5
|
+
|
|
6
|
+
==
|
|
7
|
+
|
|
8
|
+
jq Command options:
|
|
9
|
+
-n, --null-input use `null` as the single input value;
|
|
10
|
+
-R, --raw-input read each line as string instead of JSON;
|
|
11
|
+
-s, --slurp read all inputs into an array and use it as the single input value;
|
|
12
|
+
-c, --compact-output compact instead of pretty-printed output;
|
|
13
|
+
-r, --raw-output output strings without escapes and quotes;
|
|
14
|
+
--raw-output0 implies -r and output NUL after each output;
|
|
15
|
+
-j, --join-output implies -r and output without newline after each output;
|
|
16
|
+
-a, --ascii-output output strings by only ASCII characters using escape sequences;
|
|
17
|
+
-S, --sort-keys sort keys of each object on output;
|
|
18
|
+
-C, --color-output colorize JSON output;
|
|
19
|
+
-M, --monochrome-output disable colored output;
|
|
20
|
+
--tab use tabs for indentation;
|
|
21
|
+
--indent n use n spaces for indentation (max 7 spaces);
|
|
22
|
+
--unbuffered flush output stream after each output;
|
|
23
|
+
--stream parse the input value in streaming fashion;
|
|
24
|
+
--stream-errors implies --stream and report parse error as an array;
|
|
25
|
+
--seq parse input/output as application/json-seq;
|
|
26
|
+
-f, --from-file file load filter from the file;
|
|
27
|
+
-L directory search modules from the directory;
|
|
28
|
+
--arg name value set $name to the string value;
|
|
29
|
+
--argjson name value set $name to the JSON value;
|
|
30
|
+
--slurpfile name file set $name to an array of JSON values read from the file;
|
|
31
|
+
--rawfile name file set $name to string contents of file;
|
|
32
|
+
--args consume remaining arguments as positional string values;
|
|
33
|
+
--jsonargs consume remaining arguments as positional JSON values;
|
|
34
|
+
-e, --exit-status set exit status code based on the output;
|
|
35
|
+
-V, --version show the version;
|
|
36
|
+
--build-configuration show jq's build configuration;
|
|
37
|
+
-h, --help show the help;
|
|
38
|
+
-- terminates argument processing;
|
|
39
|
+
"""
|
|
40
|
+
import argparse
|
|
41
|
+
import contextlib
|
|
42
|
+
import dataclasses as dc
|
|
43
|
+
import io
|
|
44
|
+
import os
|
|
45
|
+
import subprocess
|
|
46
|
+
import sys
|
|
47
|
+
import typing as ta
|
|
48
|
+
|
|
49
|
+
from omlish import check
|
|
50
|
+
from omlish import lang
|
|
51
|
+
from omlish.funcs import pipes as fp
|
|
52
|
+
|
|
53
|
+
from .formats import FORMATS_BY_NAME
|
|
54
|
+
from .formats import Format
|
|
55
|
+
from .formats import Formats
|
|
56
|
+
from .parsing import DelimitingParser
|
|
57
|
+
from .parsing import EagerParser
|
|
58
|
+
from .parsing import StreamBuilder
|
|
59
|
+
from .parsing import StreamParser
|
|
60
|
+
from .processing import ProcessingOptions
|
|
61
|
+
from .processing import Processor
|
|
62
|
+
from .rendering import EagerRenderer
|
|
63
|
+
from .rendering import RenderingOptions
|
|
64
|
+
from .rendering import StreamRenderer
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
T = ta.TypeVar('T')
|
|
68
|
+
U = ta.TypeVar('U')
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _build_args_parser() -> argparse.ArgumentParser:
|
|
72
|
+
parser = argparse.ArgumentParser()
|
|
73
|
+
|
|
74
|
+
parser.add_argument('file', nargs='?')
|
|
75
|
+
|
|
76
|
+
parser.add_argument('--stream', action='store_true')
|
|
77
|
+
parser.add_argument('--stream-build', action='store_true')
|
|
78
|
+
|
|
79
|
+
parser.add_argument('-l', '--lines', action='store_true')
|
|
80
|
+
|
|
81
|
+
parser.add_argument('--read-buffer-size', type=int, default=0x4000)
|
|
82
|
+
|
|
83
|
+
parser.add_argument('-f', '--format')
|
|
84
|
+
|
|
85
|
+
parser.add_argument('-x', '--jmespath-expr')
|
|
86
|
+
parser.add_argument('-F', '--flat', action='store_true')
|
|
87
|
+
|
|
88
|
+
parser.add_argument('-z', '--compact', action='store_true')
|
|
89
|
+
parser.add_argument('-p', '--pretty', action='store_true')
|
|
90
|
+
parser.add_argument('-i', '--indent')
|
|
91
|
+
parser.add_argument('-s', '--sort-keys', action='store_true')
|
|
92
|
+
parser.add_argument('-R', '--raw', action='store_true')
|
|
93
|
+
parser.add_argument('-U', '--unicode', action='store_true')
|
|
94
|
+
parser.add_argument('-c', '--color', action='store_true')
|
|
95
|
+
|
|
96
|
+
parser.add_argument('-L', '--less', action='store_true')
|
|
97
|
+
|
|
98
|
+
return parser
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _parse_args(args: ta.Any = None) -> ta.Any:
|
|
102
|
+
return _build_args_parser().parse_args(args)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dc.dataclass(frozen=True, kw_only=True)
|
|
106
|
+
class RunConfiguration:
|
|
107
|
+
format: Format
|
|
108
|
+
processing: ProcessingOptions
|
|
109
|
+
rendering: RenderingOptions
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _process_args(args: ta.Any) -> RunConfiguration:
|
|
113
|
+
fmt_name = args.format
|
|
114
|
+
if fmt_name is None:
|
|
115
|
+
if args.file is not None:
|
|
116
|
+
ext = args.file.rpartition('.')[2]
|
|
117
|
+
if ext in FORMATS_BY_NAME:
|
|
118
|
+
fmt_name = ext
|
|
119
|
+
if fmt_name is None:
|
|
120
|
+
fmt_name = 'json'
|
|
121
|
+
format = FORMATS_BY_NAME[fmt_name] # noqa
|
|
122
|
+
|
|
123
|
+
if args.stream:
|
|
124
|
+
check.arg(format is Formats.JSON.value)
|
|
125
|
+
|
|
126
|
+
#
|
|
127
|
+
|
|
128
|
+
processing = ProcessingOptions(
|
|
129
|
+
jmespath_expr=args.jmespath_expr,
|
|
130
|
+
flat=args.flat,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
#
|
|
134
|
+
|
|
135
|
+
separators = None
|
|
136
|
+
if args.compact:
|
|
137
|
+
separators = (',', ':')
|
|
138
|
+
|
|
139
|
+
indent = None
|
|
140
|
+
if args.pretty:
|
|
141
|
+
indent = 2
|
|
142
|
+
if args.indent:
|
|
143
|
+
try:
|
|
144
|
+
indent = int(args.indent)
|
|
145
|
+
except ValueError:
|
|
146
|
+
indent = args.indent
|
|
147
|
+
|
|
148
|
+
rendering = RenderingOptions(
|
|
149
|
+
indent=indent,
|
|
150
|
+
separators=separators,
|
|
151
|
+
sort_keys=args.sort_keys,
|
|
152
|
+
raw=args.raw,
|
|
153
|
+
unicode=args.unicode,
|
|
154
|
+
color=args.color,
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
#
|
|
158
|
+
|
|
159
|
+
return RunConfiguration(
|
|
160
|
+
format=format,
|
|
161
|
+
processing=processing,
|
|
162
|
+
rendering=rendering,
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _main() -> None:
|
|
167
|
+
args = _parse_args()
|
|
168
|
+
|
|
169
|
+
#
|
|
170
|
+
|
|
171
|
+
cfg = _process_args(args)
|
|
172
|
+
|
|
173
|
+
#
|
|
174
|
+
|
|
175
|
+
with contextlib.ExitStack() as es:
|
|
176
|
+
if args.file is None:
|
|
177
|
+
in_file = sys.stdin.buffer
|
|
178
|
+
|
|
179
|
+
else:
|
|
180
|
+
in_file = es.enter_context(open(args.file, 'rb'))
|
|
181
|
+
|
|
182
|
+
def yield_input() -> ta.Generator[bytes, None, None]:
|
|
183
|
+
fd = check.isinstance(in_file.fileno(), int)
|
|
184
|
+
|
|
185
|
+
while True:
|
|
186
|
+
buf = os.read(fd, args.read_buffer_size)
|
|
187
|
+
|
|
188
|
+
yield buf
|
|
189
|
+
|
|
190
|
+
if not buf:
|
|
191
|
+
break
|
|
192
|
+
|
|
193
|
+
#
|
|
194
|
+
|
|
195
|
+
if args.less:
|
|
196
|
+
less = subprocess.Popen(
|
|
197
|
+
[
|
|
198
|
+
'less',
|
|
199
|
+
*(['-R'] if cfg.rendering.color else []),
|
|
200
|
+
],
|
|
201
|
+
stdin=subprocess.PIPE,
|
|
202
|
+
encoding='utf-8',
|
|
203
|
+
)
|
|
204
|
+
out = check.not_none(less.stdin)
|
|
205
|
+
|
|
206
|
+
def close_less() -> None:
|
|
207
|
+
out.close()
|
|
208
|
+
less.wait()
|
|
209
|
+
|
|
210
|
+
es.enter_context(lang.defer(close_less)) # noqa
|
|
211
|
+
|
|
212
|
+
else:
|
|
213
|
+
out = sys.stdout
|
|
214
|
+
|
|
215
|
+
#
|
|
216
|
+
|
|
217
|
+
parser: ta.Any
|
|
218
|
+
renderer: ta.Any
|
|
219
|
+
|
|
220
|
+
if args.stream:
|
|
221
|
+
with contextlib.ExitStack() as es2:
|
|
222
|
+
parser = es2.enter_context(StreamParser())
|
|
223
|
+
|
|
224
|
+
def flush_output(
|
|
225
|
+
fn: ta.Callable[[T], ta.Iterable[U]],
|
|
226
|
+
i: T,
|
|
227
|
+
) -> ta.Generator[U, None, None]:
|
|
228
|
+
n = 0
|
|
229
|
+
for o in fn(i):
|
|
230
|
+
yield o
|
|
231
|
+
n += 1
|
|
232
|
+
if n:
|
|
233
|
+
out.flush()
|
|
234
|
+
|
|
235
|
+
pipeline: ta.Any
|
|
236
|
+
|
|
237
|
+
if args.stream_build:
|
|
238
|
+
builder: StreamBuilder = es2.enter_context(StreamBuilder())
|
|
239
|
+
processor = Processor(cfg.processing)
|
|
240
|
+
renderer = EagerRenderer(cfg.rendering)
|
|
241
|
+
trailing_newline = False
|
|
242
|
+
|
|
243
|
+
def append_newlines(
|
|
244
|
+
fn: ta.Callable[[T], ta.Iterable[str]],
|
|
245
|
+
i: T,
|
|
246
|
+
) -> ta.Generator[str, None, None]:
|
|
247
|
+
yield from fn(i)
|
|
248
|
+
yield '\n'
|
|
249
|
+
|
|
250
|
+
pipeline = lambda v: (renderer.render(v),) # Any -> [str] # noqa
|
|
251
|
+
pipeline = fp.bind(append_newlines, pipeline) # Any -> [str]
|
|
252
|
+
pipeline = fp.bind(lang.flatmap, pipeline) # [Any] -> [str]
|
|
253
|
+
pipeline = fp.pipe(fp.bind(lang.flatmap, processor.process), pipeline) # [Any] -> [str]
|
|
254
|
+
pipeline = fp.pipe(fp.bind(lang.flatmap, builder.build), pipeline) # [JsonStreamParserEvent] -> [str] # noqa
|
|
255
|
+
pipeline = fp.pipe(parser.parse, pipeline) # bytes -> [str]
|
|
256
|
+
|
|
257
|
+
else:
|
|
258
|
+
renderer = StreamRenderer(cfg.rendering)
|
|
259
|
+
trailing_newline = True
|
|
260
|
+
|
|
261
|
+
pipeline = renderer.render # JsonStreamParserEvent -> [str]
|
|
262
|
+
pipeline = fp.bind(lang.flatmap, pipeline) # [JsonStreamParserEvent] -> [str]
|
|
263
|
+
pipeline = fp.pipe(parser.parse, pipeline) # bytes -> [str]
|
|
264
|
+
|
|
265
|
+
pipeline = fp.bind(flush_output, pipeline) # bytes -> [str]
|
|
266
|
+
|
|
267
|
+
for buf in yield_input():
|
|
268
|
+
for s in pipeline(buf):
|
|
269
|
+
print(s, file=out, end='')
|
|
270
|
+
|
|
271
|
+
if trailing_newline:
|
|
272
|
+
print(file=out)
|
|
273
|
+
|
|
274
|
+
elif args.lines:
|
|
275
|
+
parser = DelimitingParser(cfg.format)
|
|
276
|
+
processor = Processor(cfg.processing)
|
|
277
|
+
renderer = EagerRenderer(cfg.rendering)
|
|
278
|
+
|
|
279
|
+
for buf in yield_input():
|
|
280
|
+
for v in parser.parse(buf):
|
|
281
|
+
for e in processor.process(v):
|
|
282
|
+
s = renderer.render(e)
|
|
283
|
+
print(s, file=out)
|
|
284
|
+
|
|
285
|
+
else:
|
|
286
|
+
parser = EagerParser(cfg.format)
|
|
287
|
+
processor = Processor(cfg.processing)
|
|
288
|
+
renderer = EagerRenderer(cfg.rendering)
|
|
289
|
+
|
|
290
|
+
with io.TextIOWrapper(in_file) as tf:
|
|
291
|
+
v = parser.parse(tf)
|
|
292
|
+
|
|
293
|
+
for e in processor.process(v):
|
|
294
|
+
s = renderer.render(e)
|
|
295
|
+
print(s, file=out)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
if __name__ == '__main__':
|
|
299
|
+
_main()
|
omdev/json/formats.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TODO:
|
|
3
|
+
- options lol - csv header, newline, etc
|
|
4
|
+
"""
|
|
5
|
+
import dataclasses as dc
|
|
6
|
+
import enum
|
|
7
|
+
import json
|
|
8
|
+
import typing as ta
|
|
9
|
+
|
|
10
|
+
from omlish import lang
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
if ta.TYPE_CHECKING:
|
|
14
|
+
import ast
|
|
15
|
+
import csv
|
|
16
|
+
import tomllib
|
|
17
|
+
|
|
18
|
+
import yaml
|
|
19
|
+
|
|
20
|
+
from omlish.formats import dotenv
|
|
21
|
+
from omlish.formats import props
|
|
22
|
+
from omlish.formats import xml
|
|
23
|
+
|
|
24
|
+
else:
|
|
25
|
+
ast = lang.proxy_import('ast')
|
|
26
|
+
csv = lang.proxy_import('csv')
|
|
27
|
+
tomllib = lang.proxy_import('tomllib')
|
|
28
|
+
|
|
29
|
+
yaml = lang.proxy_import('yaml')
|
|
30
|
+
|
|
31
|
+
dotenv = lang.proxy_import('omlish.formats.dotenv')
|
|
32
|
+
props = lang.proxy_import('omlish.formats.props')
|
|
33
|
+
xml = lang.proxy_import('omlish.formats.xml')
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
##
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dc.dataclass(frozen=True)
|
|
40
|
+
class Format:
|
|
41
|
+
names: ta.Sequence[str]
|
|
42
|
+
load: ta.Callable[[ta.TextIO], ta.Any]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Formats(enum.Enum):
|
|
46
|
+
JSON = Format(['json'], json.load)
|
|
47
|
+
|
|
48
|
+
YAML = Format(['yaml', 'yml'], lambda f: yaml.safe_load(f))
|
|
49
|
+
|
|
50
|
+
TOML = Format(['toml'], lambda f: tomllib.loads(f.read()))
|
|
51
|
+
|
|
52
|
+
ENV = Format(['env', 'dotenv'], lambda f: dotenv.dotenv_values(stream=f))
|
|
53
|
+
|
|
54
|
+
PROPS = Format(['properties', 'props'], lambda f: dict(props.Properties().load(f.read())))
|
|
55
|
+
|
|
56
|
+
PY = Format(['py', 'python', 'repr'], lambda f: ast.literal_eval(f.read()))
|
|
57
|
+
|
|
58
|
+
XML = Format(['xml'], lambda f: xml.build_simple_element(xml.parse_tree(f.read()).getroot()).as_dict())
|
|
59
|
+
|
|
60
|
+
CSV = Format(['csv'], lambda f: list(csv.DictReader(f)))
|
|
61
|
+
TSV = Format(['tsv'], lambda f: list(csv.DictReader(f, delimiter='\t')))
|
|
62
|
+
FLAT_CSV = Format(['fcsv'], lambda f: list(csv.reader(f)))
|
|
63
|
+
FLAT_TSV = Format(['ftsv'], lambda f: list(csv.reader(f, delimiter='\t')))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
FORMATS_BY_NAME: ta.Mapping[str, Format] = {
|
|
67
|
+
n: f
|
|
68
|
+
for e in Formats
|
|
69
|
+
for f in [e.value]
|
|
70
|
+
for n in f.names
|
|
71
|
+
}
|
omdev/json/io.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# """
|
|
2
|
+
# TODO:
|
|
3
|
+
# - -I/-Ogz, lz4, etc
|
|
4
|
+
# - fnpairs? or not yet just do it
|
|
5
|
+
# """
|
|
6
|
+
# import abc
|
|
7
|
+
# import contextlib
|
|
8
|
+
# import dataclasses as dc
|
|
9
|
+
# import gzip
|
|
10
|
+
# import os
|
|
11
|
+
# import typing as ta
|
|
12
|
+
#
|
|
13
|
+
# from .... import lang
|
|
14
|
+
#
|
|
15
|
+
#
|
|
16
|
+
# if ta.TYPE_CHECKING:
|
|
17
|
+
# import bz2 as _bz2
|
|
18
|
+
# import gzip as _gzip
|
|
19
|
+
# import lzma as _lzma
|
|
20
|
+
# else:
|
|
21
|
+
# _bz2 = lang.proxy_import('bz2')
|
|
22
|
+
# _gzip = lang.proxy_import('gzip')
|
|
23
|
+
# _lzma = lang.proxy_import('lzma')
|
|
24
|
+
#
|
|
25
|
+
#
|
|
26
|
+
# ##
|
|
27
|
+
#
|
|
28
|
+
#
|
|
29
|
+
# class BaseIo(lang.Abstract):
|
|
30
|
+
# def close(self) -> None:
|
|
31
|
+
# raise NotImplementedError
|
|
32
|
+
#
|
|
33
|
+
# def fileno(self) -> int | None:
|
|
34
|
+
# return None
|
|
35
|
+
#
|
|
36
|
+
#
|
|
37
|
+
# class Input(BaseIo):
|
|
38
|
+
# @abc.abstractmethod
|
|
39
|
+
# def read(self, sz: int | None = None) -> bytes:
|
|
40
|
+
# raise NotImplementedError
|
|
41
|
+
#
|
|
42
|
+
#
|
|
43
|
+
# class Output(BaseIo):
|
|
44
|
+
# @abc.abstractmethod
|
|
45
|
+
# def write(self, data: bytes) -> int:
|
|
46
|
+
# raise NotImplementedError
|
|
47
|
+
#
|
|
48
|
+
#
|
|
49
|
+
# #
|
|
50
|
+
#
|
|
51
|
+
#
|
|
52
|
+
# DEFAULT_READ_SZ = 0x4000
|
|
53
|
+
#
|
|
54
|
+
#
|
|
55
|
+
# @dc.dataclass(frozen=True)
|
|
56
|
+
# class FdIo(Input, Output):
|
|
57
|
+
# fd: int
|
|
58
|
+
#
|
|
59
|
+
# default_read_sz: int = DEFAULT_READ_SZ
|
|
60
|
+
#
|
|
61
|
+
# def read(self, sz: int | None = None) -> bytes:
|
|
62
|
+
# return os.read(self.fd, sz or self.default_read_sz)
|
|
63
|
+
#
|
|
64
|
+
# def write(self, data: bytes) -> int:
|
|
65
|
+
# return os.write(self.fd, data)
|
|
66
|
+
#
|
|
67
|
+
#
|
|
68
|
+
# ##
|
|
69
|
+
#
|
|
70
|
+
#
|
|
71
|
+
# @contextlib.contextmanager
|
|
72
|
+
# def gzip_io_codec(f: ta.IO, mode: str) -> ta.ContextManager[ta.IO]:
|
|
73
|
+
# with gzip.open(f, mode) as o:
|
|
74
|
+
# yield o
|
omdev/json/parsing.py
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import codecs
|
|
2
|
+
import io
|
|
3
|
+
import typing as ta
|
|
4
|
+
|
|
5
|
+
from omlish import check
|
|
6
|
+
from omlish import lang
|
|
7
|
+
from omlish.formats.json.stream.build import JsonObjectBuilder
|
|
8
|
+
from omlish.formats.json.stream.lex import JsonStreamLexer
|
|
9
|
+
from omlish.formats.json.stream.parse import JsonStreamParser
|
|
10
|
+
from omlish.formats.json.stream.parse import JsonStreamParserEvent
|
|
11
|
+
from omlish.lite.io import DelimitingBuffer
|
|
12
|
+
|
|
13
|
+
from .formats import Format
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
##
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class EagerParser:
|
|
20
|
+
def __init__(self, fmt: Format) -> None:
|
|
21
|
+
super().__init__()
|
|
22
|
+
|
|
23
|
+
self._fmt = fmt
|
|
24
|
+
|
|
25
|
+
def parse(self, f: ta.TextIO) -> ta.Generator[ta.Any, None, None]:
|
|
26
|
+
return self._fmt.load(f)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
##
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class DelimitingParser:
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
fmt: Format,
|
|
36
|
+
*,
|
|
37
|
+
delimiters: ta.Iterable[int] = b'\n',
|
|
38
|
+
) -> None:
|
|
39
|
+
super().__init__()
|
|
40
|
+
|
|
41
|
+
self._fmt = fmt
|
|
42
|
+
|
|
43
|
+
self._db = DelimitingBuffer(delimiters)
|
|
44
|
+
|
|
45
|
+
def parse(self, b: bytes) -> ta.Generator[ta.Any, None, None]:
|
|
46
|
+
for chunk in self._db.feed(b):
|
|
47
|
+
s = check.isinstance(chunk, bytes).decode('utf-8')
|
|
48
|
+
v = self._fmt.load(io.StringIO(s))
|
|
49
|
+
yield v
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
##
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class StreamBuilder(lang.ExitStacked):
|
|
56
|
+
_builder: JsonObjectBuilder | None = None
|
|
57
|
+
|
|
58
|
+
def __enter__(self) -> ta.Self:
|
|
59
|
+
super().__enter__()
|
|
60
|
+
self._builder = self._enter_context(JsonObjectBuilder())
|
|
61
|
+
return self
|
|
62
|
+
|
|
63
|
+
def build(self, e: JsonStreamParserEvent) -> ta.Generator[ta.Any, None, None]:
|
|
64
|
+
yield from check.not_none(self._builder)(e)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class StreamParser(lang.ExitStacked):
|
|
68
|
+
_decoder: codecs.IncrementalDecoder
|
|
69
|
+
_lex: JsonStreamLexer
|
|
70
|
+
_parse: JsonStreamParser
|
|
71
|
+
|
|
72
|
+
def __enter__(self) -> ta.Self:
|
|
73
|
+
super().__enter__()
|
|
74
|
+
self._decoder = codecs.getincrementaldecoder('utf-8')()
|
|
75
|
+
self._lex = self._enter_context(JsonStreamLexer())
|
|
76
|
+
self._parse = self._enter_context(JsonStreamParser())
|
|
77
|
+
return self
|
|
78
|
+
|
|
79
|
+
def parse(self, b: bytes) -> ta.Generator[JsonStreamParserEvent, None, None]:
|
|
80
|
+
for s in self._decoder.decode(b, not b):
|
|
81
|
+
for c in s:
|
|
82
|
+
for t in self._lex(c):
|
|
83
|
+
yield from self._parse(t)
|
omdev/json/processing.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import dataclasses as dc
|
|
2
|
+
import typing as ta
|
|
3
|
+
|
|
4
|
+
from omlish import lang
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
if ta.TYPE_CHECKING:
|
|
8
|
+
from omlish.specs import jmespath
|
|
9
|
+
else:
|
|
10
|
+
jmespath = lang.proxy_import('omlish.specs.jmespath')
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
##
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dc.dataclass(frozen=True, kw_only=True)
|
|
17
|
+
class ProcessingOptions:
|
|
18
|
+
jmespath_expr: ta.Any | None = None
|
|
19
|
+
flat: bool = False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Processor:
|
|
23
|
+
def __init__(self, opts: ProcessingOptions) -> None:
|
|
24
|
+
super().__init__()
|
|
25
|
+
|
|
26
|
+
self._opts = opts
|
|
27
|
+
|
|
28
|
+
jmespath_expr = opts.jmespath_expr
|
|
29
|
+
if isinstance(jmespath_expr, str):
|
|
30
|
+
jmespath_expr = jmespath.compile(jmespath_expr)
|
|
31
|
+
self._jmespath_expr: ta.Any | None = jmespath_expr
|
|
32
|
+
|
|
33
|
+
def process(self, v: ta.Any) -> ta.Iterable[ta.Any]:
|
|
34
|
+
if self._jmespath_expr is not None:
|
|
35
|
+
v = self._jmespath_expr.search(v)
|
|
36
|
+
|
|
37
|
+
if self._opts.flat:
|
|
38
|
+
if (
|
|
39
|
+
not isinstance(v, ta.Iterable) or # noqa
|
|
40
|
+
isinstance(v, ta.Mapping) or
|
|
41
|
+
isinstance(v, lang.BUILTIN_SCALAR_ITERABLE_TYPES)
|
|
42
|
+
):
|
|
43
|
+
raise TypeError(f'Flat output must be flat collections, got {type(v)}', v)
|
|
44
|
+
|
|
45
|
+
yield from v
|
|
46
|
+
|
|
47
|
+
else:
|
|
48
|
+
yield v
|
omdev/json/rendering.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import dataclasses as dc
|
|
2
|
+
import json
|
|
3
|
+
import typing as ta
|
|
4
|
+
|
|
5
|
+
from omlish import lang
|
|
6
|
+
from omlish import term
|
|
7
|
+
from omlish.formats.json.render import JsonRenderer
|
|
8
|
+
from omlish.formats.json.stream.parse import JsonStreamParserEvent
|
|
9
|
+
from omlish.formats.json.stream.render import StreamJsonRenderer
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
##
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dc.dataclass(frozen=True, kw_only=True)
|
|
16
|
+
class RenderingOptions:
|
|
17
|
+
indent: int | str | None = None
|
|
18
|
+
separators: tuple[str, str] | None = None
|
|
19
|
+
sort_keys: bool = False
|
|
20
|
+
raw: bool = False
|
|
21
|
+
unicode: bool = False
|
|
22
|
+
color: bool = False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def make_render_kwargs(opts: RenderingOptions) -> ta.Mapping[str, ta.Any]:
|
|
26
|
+
return dict(
|
|
27
|
+
indent=opts.indent,
|
|
28
|
+
separators=opts.separators,
|
|
29
|
+
sort_keys=opts.sort_keys,
|
|
30
|
+
ensure_ascii=not opts.unicode,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Renderer(lang.Abstract):
|
|
35
|
+
def __init__(self, opts: RenderingOptions) -> None:
|
|
36
|
+
super().__init__()
|
|
37
|
+
self._opts = opts
|
|
38
|
+
self._kw = make_render_kwargs(opts)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
##
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def term_color(o: ta.Any, state: JsonRenderer.State) -> tuple[str, str]:
|
|
45
|
+
if state is JsonRenderer.State.KEY:
|
|
46
|
+
return term.SGR(term.SGRs.FG.BRIGHT_BLUE), term.SGR(term.SGRs.RESET)
|
|
47
|
+
elif isinstance(o, str):
|
|
48
|
+
return term.SGR(term.SGRs.FG.GREEN), term.SGR(term.SGRs.RESET)
|
|
49
|
+
else:
|
|
50
|
+
return '', ''
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
##
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class EagerRenderer(Renderer):
|
|
57
|
+
def render(self, v: ta.Any) -> str:
|
|
58
|
+
if self._opts.raw:
|
|
59
|
+
if not isinstance(v, str):
|
|
60
|
+
raise TypeError(f'Raw output must be strings, got {type(v)}', v)
|
|
61
|
+
|
|
62
|
+
return v
|
|
63
|
+
|
|
64
|
+
elif self._opts.color:
|
|
65
|
+
return JsonRenderer.render_str(
|
|
66
|
+
v,
|
|
67
|
+
**self._kw,
|
|
68
|
+
style=term_color,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
else:
|
|
72
|
+
return json.dumps(
|
|
73
|
+
v,
|
|
74
|
+
**self._kw,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
##
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class StreamRenderer(Renderer):
|
|
82
|
+
def __init__(self, opts: RenderingOptions) -> None:
|
|
83
|
+
super().__init__(opts)
|
|
84
|
+
|
|
85
|
+
self._renderer = StreamJsonRenderer(
|
|
86
|
+
style=term_color if self._opts.color else None,
|
|
87
|
+
delimiter='\n',
|
|
88
|
+
**self._kw,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
def render(self, e: JsonStreamParserEvent) -> ta.Generator[str, None, None]:
|
|
92
|
+
return self._renderer.render((e,))
|
omdev/packaging/marshal.py
CHANGED
|
@@ -7,8 +7,8 @@ import typing as ta
|
|
|
7
7
|
|
|
8
8
|
from omlish import lang
|
|
9
9
|
from omlish import marshal as msh
|
|
10
|
-
from omlish import matchfns as mfs
|
|
11
10
|
from omlish import reflect as rfl
|
|
11
|
+
from omlish.funcs import match as mfs
|
|
12
12
|
|
|
13
13
|
from .requires import RequiresMarkerItem
|
|
14
14
|
from .requires import RequiresMarkerList
|
omdev/scripts/pyproject.py
CHANGED
|
@@ -3207,6 +3207,13 @@ class PolymorphicObjMarshaler(ObjMarshaler):
|
|
|
3207
3207
|
impls_by_ty: ta.Mapping[type, Impl]
|
|
3208
3208
|
impls_by_tag: ta.Mapping[str, Impl]
|
|
3209
3209
|
|
|
3210
|
+
@classmethod
|
|
3211
|
+
def of(cls, impls: ta.Iterable[Impl]) -> 'PolymorphicObjMarshaler':
|
|
3212
|
+
return cls(
|
|
3213
|
+
{i.ty: i for i in impls},
|
|
3214
|
+
{i.tag: i for i in impls},
|
|
3215
|
+
)
|
|
3216
|
+
|
|
3210
3217
|
def marshal(self, o: ta.Any) -> ta.Any:
|
|
3211
3218
|
impl = self.impls_by_ty[type(o)]
|
|
3212
3219
|
return {impl.tag: impl.m.marshal(o)}
|
|
@@ -3294,7 +3301,7 @@ def _make_obj_marshaler(
|
|
|
3294
3301
|
) -> ObjMarshaler:
|
|
3295
3302
|
if isinstance(ty, type):
|
|
3296
3303
|
if abc.ABC in ty.__bases__:
|
|
3297
|
-
|
|
3304
|
+
return PolymorphicObjMarshaler.of([ # type: ignore
|
|
3298
3305
|
PolymorphicObjMarshaler.Impl(
|
|
3299
3306
|
ity,
|
|
3300
3307
|
ity.__qualname__,
|
|
@@ -3302,11 +3309,7 @@ def _make_obj_marshaler(
|
|
|
3302
3309
|
)
|
|
3303
3310
|
for ity in deep_subclasses(ty)
|
|
3304
3311
|
if abc.ABC not in ity.__bases__
|
|
3305
|
-
]
|
|
3306
|
-
return PolymorphicObjMarshaler(
|
|
3307
|
-
{i.ty: i for i in impls},
|
|
3308
|
-
{i.tag: i for i in impls},
|
|
3309
|
-
)
|
|
3312
|
+
])
|
|
3310
3313
|
|
|
3311
3314
|
if issubclass(ty, enum.Enum):
|
|
3312
3315
|
return EnumObjMarshaler(ty)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: omdev
|
|
3
|
-
Version: 0.0.0.
|
|
3
|
+
Version: 0.0.0.dev137
|
|
4
4
|
Summary: omdev
|
|
5
5
|
Author: wrmsr
|
|
6
6
|
License: BSD-3-Clause
|
|
@@ -12,7 +12,7 @@ Classifier: Operating System :: OS Independent
|
|
|
12
12
|
Classifier: Operating System :: POSIX
|
|
13
13
|
Requires-Python: >=3.12
|
|
14
14
|
License-File: LICENSE
|
|
15
|
-
Requires-Dist: omlish==0.0.0.
|
|
15
|
+
Requires-Dist: omlish==0.0.0.dev137
|
|
16
16
|
Provides-Extra: all
|
|
17
17
|
Requires-Dist: black~=24.10; extra == "all"
|
|
18
18
|
Requires-Dist: pycparser~=2.22; extra == "all"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
omdev/.manifests.json,sha256=
|
|
1
|
+
omdev/.manifests.json,sha256=AQ_MtyaYpRcm1jSGJIDiCxxwm5CnW3wx9TcjzruR8WY,7769
|
|
2
2
|
omdev/__about__.py,sha256=n5x-SO70OgbDQFzQ1d7sZDVMsnkQc4PxQZPFaIQFa0E,1281
|
|
3
3
|
omdev/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
omdev/bracepy.py,sha256=I8EdqtDvxzAi3I8TuMEW-RBfwXfqKbwp06CfOdj3L1o,2743
|
|
@@ -79,6 +79,14 @@ omdev/interp/resolvers.py,sha256=tpzlmqGp1C4QKdA6TfcPmtmaygu7mb6WK2RPSbyNQ6s,302
|
|
|
79
79
|
omdev/interp/standalone.py,sha256=XcltiL7ypcfV89C82_3knQ3Kx7aW4wnnxf2056ZXC3A,7731
|
|
80
80
|
omdev/interp/system.py,sha256=bI-JhX4GVJqW7wMxnIa-DGJWnCLmFcIsnl9pc1RGY2g,3513
|
|
81
81
|
omdev/interp/types.py,sha256=EMN3StEMkFoQAMUIZd7JYL4uUWqzFGY-2hTL8EBznYM,2437
|
|
82
|
+
omdev/json/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
83
|
+
omdev/json/__main__.py,sha256=A2NlzsXebLuA0Zw6sikwM_tXZsyBYf33Q8qpNhtaY5o,167
|
|
84
|
+
omdev/json/cli.py,sha256=EubIMT-n2XsjWBZjSy2fWXqijlwrIhLsfbkg3SZzi28,9586
|
|
85
|
+
omdev/json/formats.py,sha256=XHabWAQnAQ9uNS7fHu3mdgxoptWw00RVLq08uadD6kk,1753
|
|
86
|
+
omdev/json/io.py,sha256=IoJ5asjS_gUan5TcrGgS0KpJaiSE5YQgEGljFMCcDto,1427
|
|
87
|
+
omdev/json/parsing.py,sha256=QLsdYv-fRFNIWwy8KLr19fQ5RodiQEX2hIw4jndnEXM,2180
|
|
88
|
+
omdev/json/processing.py,sha256=iFm5VqaxJ97WHaun2ed7NEjMxhFeJqf28bLNfoDJft0,1209
|
|
89
|
+
omdev/json/rendering.py,sha256=jNShMfCpFR9-Kcn6cUFuOChXHjg71diuTC4x7Ofmz-o,2240
|
|
82
90
|
omdev/magic/__init__.py,sha256=CBzRB71RLyylkrj8dph6JUEddA8KSMJvDgriHqFfJGU,478
|
|
83
91
|
omdev/magic/find.py,sha256=tTmpWXAleaXG3_kNOsRF7s8D0CpYMXbdz6-HbCNBW90,7070
|
|
84
92
|
omdev/magic/magic.py,sha256=h1nxoW6CV1MRCiHjDt3sO4kmG0qTtTRbkDNiPLGo2BE,224
|
|
@@ -91,7 +99,7 @@ omdev/manifests/types.py,sha256=Jv6PAdVLPb9Hh4y6vDhPlWuMNBBViin1bC_u83jfsH4,234
|
|
|
91
99
|
omdev/mypy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
92
100
|
omdev/mypy/debug.py,sha256=WcZw-3Z1njg_KFGqi3DB6RuqbBa3dLArJnjVCuY1Mn0,3003
|
|
93
101
|
omdev/packaging/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
94
|
-
omdev/packaging/marshal.py,sha256=
|
|
102
|
+
omdev/packaging/marshal.py,sha256=c4Mdy5-s4O51eEKMWWBwkhTGQ1FoxVI2lD6CxI_cF3k,2379
|
|
95
103
|
omdev/packaging/names.py,sha256=-a7AykFPVR1i6EYJepbe3ABRrZQ_tPPmK5olzbn9HLI,2528
|
|
96
104
|
omdev/packaging/requires.py,sha256=kyyVCM0RyaKqmXd6tU5zf0QjWvCLQMNV_ev5hvOee_o,15731
|
|
97
105
|
omdev/packaging/specifiers.py,sha256=X8xOcwRLXTQYx5mYAS3ijqoTLlCtYESyUu4fX9bvblY,17447
|
|
@@ -123,7 +131,7 @@ omdev/scripts/execrss.py,sha256=mR0G0wERBYtQmVIn63lCIIFb5zkCM6X_XOENDFYDBKc,651
|
|
|
123
131
|
omdev/scripts/exectime.py,sha256=sFb376GflU6s9gNX-2-we8hgH6w5MuQNS9g6i4SqJIo,610
|
|
124
132
|
omdev/scripts/importtrace.py,sha256=oa7CtcWJVMNDbyIEiRHej6ICfABfErMeo4_haIqe18Q,14041
|
|
125
133
|
omdev/scripts/interp.py,sha256=8R3jVdmf8Fyl8B4KI-dctatXmFHEAAzsXpNRiCDVdqQ,73800
|
|
126
|
-
omdev/scripts/pyproject.py,sha256=
|
|
134
|
+
omdev/scripts/pyproject.py,sha256=TXTlKLyTb9Auj5kJMhhGoec5Yi-hvYOi_WnZJH9E0xQ,175193
|
|
127
135
|
omdev/scripts/slowcat.py,sha256=lssv4yrgJHiWfOiHkUut2p8E8Tq32zB-ujXESQxFFHY,2728
|
|
128
136
|
omdev/scripts/tmpexec.py,sha256=WTYcf56Tj2qjYV14AWmV8SfT0u6Y8eIU6cKgQRvEK3c,1442
|
|
129
137
|
omdev/toml/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
|
|
@@ -143,9 +151,9 @@ omdev/tools/sqlrepl.py,sha256=tmFZh80-xsGM62dyQ7_UGLebChrj7IHbIPYBWDJMgVk,5741
|
|
|
143
151
|
omdev/tools/pawk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
144
152
|
omdev/tools/pawk/__main__.py,sha256=VCqeRVnqT1RPEoIrqHFSu4PXVMg4YEgF4qCQm90-eRI,66
|
|
145
153
|
omdev/tools/pawk/pawk.py,sha256=Eckymn22GfychCQcQi96BFqRo_LmiJ-EPhC8TTUJdB4,11446
|
|
146
|
-
omdev-0.0.0.
|
|
147
|
-
omdev-0.0.0.
|
|
148
|
-
omdev-0.0.0.
|
|
149
|
-
omdev-0.0.0.
|
|
150
|
-
omdev-0.0.0.
|
|
151
|
-
omdev-0.0.0.
|
|
154
|
+
omdev-0.0.0.dev137.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
|
|
155
|
+
omdev-0.0.0.dev137.dist-info/METADATA,sha256=Zi_XDG9wqGE2_OQQ9Iu1BRx9nOHKOhQj-L8X05MS42M,1760
|
|
156
|
+
omdev-0.0.0.dev137.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
157
|
+
omdev-0.0.0.dev137.dist-info/entry_points.txt,sha256=dHLXFmq5D9B8qUyhRtFqTGWGxlbx3t5ejedjrnXNYLU,33
|
|
158
|
+
omdev-0.0.0.dev137.dist-info/top_level.txt,sha256=1nr7j30fEWgLYHW3lGR9pkdHkb7knv1U1ES1XRNVQ6k,6
|
|
159
|
+
omdev-0.0.0.dev137.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|