terset 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- terset/__init__.py +5 -0
- terset/_terset.py +514 -0
- terset/py.typed +0 -0
- terset-0.2.1.dist-info/METADATA +132 -0
- terset-0.2.1.dist-info/RECORD +8 -0
- terset-0.2.1.dist-info/WHEEL +5 -0
- terset-0.2.1.dist-info/licenses/LICENSE +21 -0
- terset-0.2.1.dist-info/top_level.txt +1 -0
terset/__init__.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""Encode and decode canonical Terset text without runtime dependencies."""
|
|
2
|
+
|
|
3
|
+
from ._terset import RULES, JsonInputRoot, JsonInputValue, JsonObject, JsonRoot, JsonValue, decode, encode, rules_for
|
|
4
|
+
|
|
5
|
+
__all__ = ["RULES", "JsonInputRoot", "JsonInputValue", "JsonObject", "JsonRoot", "JsonValue", "decode", "encode", "rules_for"]
|
terset/_terset.py
ADDED
|
@@ -0,0 +1,514 @@
|
|
|
1
|
+
"""The Terset format, with JavaScript binary64 number semantics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import re
|
|
8
|
+
from collections.abc import Mapping, Sequence
|
|
9
|
+
from typing import TypeAlias, cast
|
|
10
|
+
|
|
11
|
+
JsonValue: TypeAlias = "None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue]"
|
|
12
|
+
JsonObject: TypeAlias = dict[str, JsonValue]
|
|
13
|
+
JsonRoot: TypeAlias = "JsonObject | list[JsonValue]"
|
|
14
|
+
JsonInputValue: TypeAlias = "None | bool | int | float | str | Sequence[JsonInputValue] | Mapping[str, JsonInputValue]"
|
|
15
|
+
JsonInputRoot: TypeAlias = "Mapping[str, JsonInputValue] | Sequence[JsonInputValue]"
|
|
16
|
+
|
|
17
|
+
_MAX_JSON_DEPTH = 500
|
|
18
|
+
_MAX_HEADER_SEGMENTS = 8
|
|
19
|
+
_DEPTH_ERROR = "Maximum JSON depth is 500."
|
|
20
|
+
_HeaderTrie: TypeAlias = "dict[str, _HeaderTrie | None]"
|
|
21
|
+
|
|
22
|
+
RULES = "Tables use tab-separated headers and rows. Empty cells mean absent. JSON quotes distinguish numeric, boolean or null text from scalars. Nested cells use JSON. A leading bare id header marks string IDs. name{N} has N entries; the @key column holds each entry's key and is not a field. Header a.b is field b in object a."
|
|
23
|
+
|
|
24
|
+
_SENTENCES = RULES.split(". ")
|
|
25
|
+
_BASE_RULES = ". ".join(_SENTENCES[:5]) + "."
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def rules_for(text: str) -> str:
|
|
29
|
+
"""Return the rules needed by canonical text, with decode's validation."""
|
|
30
|
+
features: set[str] = set()
|
|
31
|
+
_read_document(text, features)
|
|
32
|
+
sentences = [_BASE_RULES]
|
|
33
|
+
if "keyed" in features:
|
|
34
|
+
sentences.append(_SENTENCES[5] + ".")
|
|
35
|
+
if "nested" in features:
|
|
36
|
+
sentences.append(_SENTENCES[6])
|
|
37
|
+
return " ".join(sentences)
|
|
38
|
+
|
|
39
|
+
_KEY = re.compile(r"[A-Za-z_$][A-Za-z0-9_$.-]*\Z")
|
|
40
|
+
_NUMBER = re.compile(r"[-+]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][-+]?[0-9]+)?\Z")
|
|
41
|
+
_ENTRY = re.compile(r'^("(?:[^"\\]|\\.)*"|[A-Za-z_$][A-Za-z0-9_$.-]*)(?:\[([0-9]+)\]|\{([0-9]+)\})?:([\s\S]*)\Z')
|
|
42
|
+
_TROUBLE = re.compile(r"[\x00-\x1f\x7f\x85\u2028\u2029\ud800-\udfff]")
|
|
43
|
+
_WHITESPACE = "\u0009\u000a\u000b\u000c\u000d\u0020\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff"
|
|
44
|
+
_WORDS = {"true", "false", "null", "undefined", "NaN", "Infinity", "-Infinity"}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _unicode(text: str) -> str:
|
|
48
|
+
# Python normally stores astral characters as one code point. Accept
|
|
49
|
+
# explicit UTF-16 pairs too, keeping unpaired surrogates unchanged.
|
|
50
|
+
return re.sub(r"[\ud800-\udbff][\udc00-\udfff]", lambda match: chr(0x10000 + ((ord(match[0][0]) - 0xD800) << 10) + ord(match[0][1]) - 0xDC00), text)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _bare(text: str) -> bool:
|
|
54
|
+
text = _unicode(text)
|
|
55
|
+
return bool(text) and text.strip(_WHITESPACE) == text and not _TROUBLE.search(text) and text[0] not in '\"{[' and not _NUMBER.fullmatch(text) and text not in _WORDS
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _quote(text: str) -> str:
|
|
59
|
+
# ensure_ascii=False keeps astral and non-ASCII text, like JSON.stringify.
|
|
60
|
+
# Only lone surrogate code units must be escaped for UTF-8 transport.
|
|
61
|
+
quoted = json.dumps(_unicode(text), ensure_ascii=False)
|
|
62
|
+
return re.sub(r"[\ud800-\udfff]", lambda match: "\\u%04x" % ord(match[0]), quoted)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _keys(value: JsonObject) -> list[str]:
|
|
66
|
+
# Object.keys puts canonical array-index keys before other keys.
|
|
67
|
+
indices = [key for key in value if key.isascii() and key.isdigit() and (key == "0" or not key.startswith("0")) and len(key) <= 10 and int(key) < 4294967295]
|
|
68
|
+
index_set = set(indices)
|
|
69
|
+
return sorted(indices, key=int) + [key for key in value if key not in index_set]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _number(value: int | float) -> str:
|
|
73
|
+
number = float(value)
|
|
74
|
+
if number == 0:
|
|
75
|
+
return "0"
|
|
76
|
+
sign = "-" if number < 0 else ""
|
|
77
|
+
text = repr(abs(number)).lower()
|
|
78
|
+
mantissa, _, exponent = text.partition("e")
|
|
79
|
+
before, _, after = mantissa.partition(".")
|
|
80
|
+
digits = (before + after).lstrip("0")
|
|
81
|
+
power = (int(exponent) if exponent else 0) - len(after)
|
|
82
|
+
digits = digits.rstrip("0")
|
|
83
|
+
# repr provides the shortest round-tripping significant digits. Both
|
|
84
|
+
# runtimes choose the nearest decimal, with even ties. Only placement
|
|
85
|
+
# of the decimal point and exponent notation differ.
|
|
86
|
+
decimal_position = len((before + after).lstrip("0")) + power
|
|
87
|
+
if 0 < decimal_position <= 21:
|
|
88
|
+
body = digits[:decimal_position] + ("." + digits[decimal_position:] if decimal_position < len(digits) else "0" * (decimal_position - len(digits)))
|
|
89
|
+
elif -6 < decimal_position <= 0:
|
|
90
|
+
body = "0." + "0" * -decimal_position + digits
|
|
91
|
+
else:
|
|
92
|
+
exp = decimal_position - 1
|
|
93
|
+
body = digits[0] + ("." + digits[1:] if len(digits) > 1 else "") + "e" + ("+" if exp >= 0 else "-") + str(abs(exp))
|
|
94
|
+
return sign + body
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _validate(value: object) -> None:
|
|
98
|
+
ancestors: set[int] = set()
|
|
99
|
+
stack: list[tuple[object, int, bool]] = [(value, 0, False)]
|
|
100
|
+
while stack:
|
|
101
|
+
item, depth, leaving = stack.pop()
|
|
102
|
+
if leaving:
|
|
103
|
+
ancestors.remove(id(item))
|
|
104
|
+
continue
|
|
105
|
+
if depth > _MAX_JSON_DEPTH:
|
|
106
|
+
raise TypeError(_DEPTH_ERROR)
|
|
107
|
+
if item is None or type(item) in (str, bool):
|
|
108
|
+
continue
|
|
109
|
+
if type(item) is int or type(item) is float:
|
|
110
|
+
try:
|
|
111
|
+
number = float(item)
|
|
112
|
+
except OverflowError as error:
|
|
113
|
+
raise TypeError("Numbers must be finite.") from error
|
|
114
|
+
if not math.isfinite(number) or (number == 0 and math.copysign(1, number) < 0):
|
|
115
|
+
raise TypeError("Numbers must be finite and must not be negative zero.")
|
|
116
|
+
if type(item) is int and int(number) != item:
|
|
117
|
+
raise TypeError("Integers must be exactly representable as binary64 numbers.")
|
|
118
|
+
continue
|
|
119
|
+
if type(item) not in (list, dict):
|
|
120
|
+
raise TypeError("Expected plain JSON data.")
|
|
121
|
+
if id(item) in ancestors:
|
|
122
|
+
raise TypeError("Cyclic JSON data.")
|
|
123
|
+
ancestors.add(id(item))
|
|
124
|
+
stack.append((item, depth, True))
|
|
125
|
+
if isinstance(item, dict):
|
|
126
|
+
if any(type(key) is not str for key in item):
|
|
127
|
+
raise TypeError("JSON object keys must be strings.")
|
|
128
|
+
if len({_unicode(key) for key in item}) != len(item):
|
|
129
|
+
raise TypeError("Object keys must be distinct Unicode strings.")
|
|
130
|
+
stack.extend((child, depth + 1, False) for child in item.values())
|
|
131
|
+
else:
|
|
132
|
+
stack.extend((child, depth + 1, False) for child in cast(list[object], item))
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _json(value: JsonValue) -> str:
|
|
136
|
+
# Explicit stacks keep the supported depth independent of Python's
|
|
137
|
+
# recursion limit. Separators are emitted as tokens, never as values.
|
|
138
|
+
parts: list[str] = []
|
|
139
|
+
stack: list[tuple[bool, JsonValue]] = [(False, value)]
|
|
140
|
+
while stack:
|
|
141
|
+
literal, item = stack.pop()
|
|
142
|
+
if literal:
|
|
143
|
+
parts.append(cast(str, item))
|
|
144
|
+
elif item is None:
|
|
145
|
+
parts.append("null")
|
|
146
|
+
elif isinstance(item, bool):
|
|
147
|
+
parts.append("true" if item else "false")
|
|
148
|
+
elif isinstance(item, (int, float)):
|
|
149
|
+
parts.append(_number(item))
|
|
150
|
+
elif isinstance(item, str):
|
|
151
|
+
parts.append(_quote(item))
|
|
152
|
+
else:
|
|
153
|
+
tokens: list[tuple[bool, JsonValue]] = []
|
|
154
|
+
if isinstance(item, list):
|
|
155
|
+
parts.append("[")
|
|
156
|
+
for index, child in enumerate(item):
|
|
157
|
+
if index:
|
|
158
|
+
tokens.append((True, ","))
|
|
159
|
+
tokens.append((False, child))
|
|
160
|
+
tokens.append((True, "]"))
|
|
161
|
+
else:
|
|
162
|
+
parts.append("{")
|
|
163
|
+
for index, key in enumerate(_keys(item)):
|
|
164
|
+
if index:
|
|
165
|
+
tokens.append((True, ","))
|
|
166
|
+
tokens.extend([(True, _quote(key) + ":"), (False, item[key])])
|
|
167
|
+
tokens.append((True, "}"))
|
|
168
|
+
stack.extend(reversed(tokens))
|
|
169
|
+
return "".join(parts)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _key(text: str) -> str:
|
|
173
|
+
return text if _KEY.fullmatch(text) else _quote(text)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _cell(value: JsonValue) -> str:
|
|
177
|
+
return _unicode(value) if isinstance(value, str) and _bare(value) else _json(value)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _safe_id(text: str) -> bool:
|
|
181
|
+
text = _unicode(text)
|
|
182
|
+
return bool(text) and text.strip(_WHITESPACE) == text and not _TROUBLE.search(text)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _segment(key: str) -> bool:
|
|
186
|
+
return bool(_KEY.fullmatch(key)) and "." not in key
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
_MISSING = object()
|
|
190
|
+
_Column: TypeAlias = "tuple[list[str], list[JsonValue | object]]"
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _flatten(path: list[str], values: list[JsonValue | object]) -> list[_Column]:
|
|
194
|
+
first = values[0]
|
|
195
|
+
if len(path) >= _MAX_HEADER_SEGMENTS or not all(_segment(key) for key in path) or not isinstance(first, dict):
|
|
196
|
+
return [(path, values)]
|
|
197
|
+
keys = _keys(first)
|
|
198
|
+
if not keys or not all(_segment(key) for key in keys) or not all(
|
|
199
|
+
isinstance(value, dict) and _keys(value) == keys for value in values
|
|
200
|
+
):
|
|
201
|
+
return [(path, values)]
|
|
202
|
+
columns: list[_Column] = []
|
|
203
|
+
for key in keys:
|
|
204
|
+
columns.extend(_flatten(path + [key], [cast(JsonObject, value)[key] for value in values]))
|
|
205
|
+
return columns
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _table(value: list[JsonObject], keyed: bool, entry_keys: list[str] | None = None, max_bytes: int | float = math.inf) -> str | None:
|
|
209
|
+
if len(value) < 2:
|
|
210
|
+
return None
|
|
211
|
+
rows = [{_unicode(key): child for key, child in row.items()} for row in value]
|
|
212
|
+
fields: list[str] = []
|
|
213
|
+
positions: dict[str, int] = {}
|
|
214
|
+
for row in rows:
|
|
215
|
+
previous = -1
|
|
216
|
+
for key in _keys(row):
|
|
217
|
+
if key not in positions:
|
|
218
|
+
positions[key] = len(fields)
|
|
219
|
+
fields.append(key)
|
|
220
|
+
index = positions[key]
|
|
221
|
+
if index <= previous:
|
|
222
|
+
return None
|
|
223
|
+
previous = index
|
|
224
|
+
if not fields:
|
|
225
|
+
return None
|
|
226
|
+
if len(rows) * (len(fields) - 1) >= max_bytes:
|
|
227
|
+
return None
|
|
228
|
+
columns: list[_Column] = []
|
|
229
|
+
for field in fields:
|
|
230
|
+
columns.extend(_flatten([field], [row.get(field, _MISSING) for row in rows]))
|
|
231
|
+
ids = not keyed and len(columns[0][0]) == 1 and fields[0] == "id" and all(
|
|
232
|
+
_keys(row)[:1] == ["id"] and isinstance(row.get("id"), str)
|
|
233
|
+
and _safe_id(cast(str, row["id"]))
|
|
234
|
+
and (len(columns) != 1 or not _ENTRY.fullmatch(cast(str, row["id"]))) for row in rows
|
|
235
|
+
)
|
|
236
|
+
headings = []
|
|
237
|
+
for index, (path, _) in enumerate(columns):
|
|
238
|
+
field = path[0]
|
|
239
|
+
headings.append(".".join(path) if len(path) > 1 else _quote(field)
|
|
240
|
+
if "." in field or (index == 0 and field == "id" and not ids and not keyed) else _key(field))
|
|
241
|
+
if keyed:
|
|
242
|
+
headings.insert(0, "@key")
|
|
243
|
+
lines = ["\t".join(headings)]
|
|
244
|
+
for row_index in range(len(rows)):
|
|
245
|
+
cells: list[str] = []
|
|
246
|
+
for index, (_, values) in enumerate(columns):
|
|
247
|
+
entry = values[row_index]
|
|
248
|
+
if entry is _MISSING:
|
|
249
|
+
cells.append("")
|
|
250
|
+
elif index == 0 and ids:
|
|
251
|
+
cells.append(_unicode(cast(str, entry)))
|
|
252
|
+
elif not keyed and len(columns) == 1 and isinstance(entry, str) and _ENTRY.fullmatch(entry):
|
|
253
|
+
cells.append(_quote(entry))
|
|
254
|
+
else:
|
|
255
|
+
cells.append(_cell(cast(JsonValue, entry)))
|
|
256
|
+
if keyed:
|
|
257
|
+
if entry_keys is None:
|
|
258
|
+
raise TypeError("Missing entry keys.")
|
|
259
|
+
cells.insert(0, _unicode(entry_keys[row_index]))
|
|
260
|
+
lines.append("\t".join(cells))
|
|
261
|
+
return "\n".join(lines)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _list_table(value: JsonValue) -> str | None:
|
|
265
|
+
if not isinstance(value, list) or not all(isinstance(row, dict) for row in value):
|
|
266
|
+
return None
|
|
267
|
+
size = len(_json(value).encode("utf-8"))
|
|
268
|
+
text = _table(cast(list[JsonObject], value), False, max_bytes=size)
|
|
269
|
+
return text if text is not None and len(text.encode("utf-8")) < size else None
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def encode(value: JsonInputRoot) -> str:
|
|
273
|
+
"""Encode a JSON object or list. Invalid data raises TypeError.
|
|
274
|
+
|
|
275
|
+
Numbers use JavaScript binary64 semantics. Integers must be exactly
|
|
276
|
+
representable as binary64; negative zero and non-finite values fail.
|
|
277
|
+
"""
|
|
278
|
+
_validate(value)
|
|
279
|
+
# Validation restricts the read-only input interface to JSON's concrete
|
|
280
|
+
# built-in containers. The decoder always returns those concrete types.
|
|
281
|
+
root = cast(JsonRoot, value)
|
|
282
|
+
if isinstance(root, list):
|
|
283
|
+
table = _list_table(root)
|
|
284
|
+
return f"[{len(root)}]\n{table}" if table is not None else _json(root)
|
|
285
|
+
if not isinstance(root, dict):
|
|
286
|
+
raise TypeError("Terset requires a root object or array.")
|
|
287
|
+
if not root:
|
|
288
|
+
return "{}"
|
|
289
|
+
lines = []
|
|
290
|
+
for key in _keys(root):
|
|
291
|
+
entry = root[key]
|
|
292
|
+
name = _key(key)
|
|
293
|
+
table = _list_table(entry)
|
|
294
|
+
if table is not None and isinstance(entry, list):
|
|
295
|
+
lines.append(f"{name}[{len(entry)}]:\n{table}")
|
|
296
|
+
continue
|
|
297
|
+
fallback = name + ": " + _cell(entry)
|
|
298
|
+
if isinstance(entry, dict):
|
|
299
|
+
keys = _keys(entry)
|
|
300
|
+
rows = [entry[entry_key] for entry_key in keys]
|
|
301
|
+
if len(keys) >= 2 and all(_safe_id(entry_key) and not _ENTRY.fullmatch(entry_key) for entry_key in keys) and all(isinstance(row, dict) for row in rows):
|
|
302
|
+
body = _table(cast(list[JsonObject], rows), True, keys, len(fallback.encode("utf-8")))
|
|
303
|
+
if body is not None:
|
|
304
|
+
candidate = f"{name}{{{len(keys)}}}:\n{body}"
|
|
305
|
+
if len(candidate.encode("utf-8")) < len(fallback.encode("utf-8")):
|
|
306
|
+
lines.append(candidate)
|
|
307
|
+
continue
|
|
308
|
+
lines.append(fallback)
|
|
309
|
+
return "\n".join(lines)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _pairs(pairs: list[tuple[str, JsonValue]]) -> JsonObject:
|
|
313
|
+
result: JsonObject = {}
|
|
314
|
+
for key, value in pairs:
|
|
315
|
+
_set(result, key, value)
|
|
316
|
+
return result
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _parse_integer(text: str) -> int | float:
|
|
320
|
+
if text == "-0":
|
|
321
|
+
return -0.0
|
|
322
|
+
# All integers through 2**53 are exactly representable by binary64.
|
|
323
|
+
# Longer tokens still follow JavaScript's binary64 rounding.
|
|
324
|
+
if len(text.lstrip("-")) <= 16:
|
|
325
|
+
value = int(text)
|
|
326
|
+
if abs(value) <= 2**53:
|
|
327
|
+
return value
|
|
328
|
+
return float(text)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _reject_constant(text: str) -> JsonValue:
|
|
332
|
+
raise ValueError("Invalid JSON constant: " + text)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _parse(text: str) -> JsonValue:
|
|
336
|
+
try:
|
|
337
|
+
value = cast(JsonValue, json.loads(text, parse_int=_parse_integer, parse_float=float, parse_constant=_reject_constant, object_pairs_hook=_pairs))
|
|
338
|
+
_validate(value)
|
|
339
|
+
except (ValueError, TypeError, RecursionError) as error:
|
|
340
|
+
raise ValueError("Invalid JSON cell: " + str(error)) from error
|
|
341
|
+
return value
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _read_cell(text: str) -> JsonValue:
|
|
345
|
+
if not text:
|
|
346
|
+
raise ValueError("Empty values are allowed only as absent table cells.")
|
|
347
|
+
if text[0] in '\"{[' or text in {"true", "false", "null"} or _NUMBER.fullmatch(text):
|
|
348
|
+
return _parse(text)
|
|
349
|
+
if not _bare(text):
|
|
350
|
+
raise ValueError("Invalid bare text.")
|
|
351
|
+
return text
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def _read_key(text: str) -> str:
|
|
355
|
+
if _KEY.fullmatch(text):
|
|
356
|
+
return text
|
|
357
|
+
if not text.startswith('"'):
|
|
358
|
+
raise ValueError("Invalid field name.")
|
|
359
|
+
value = _parse(text)
|
|
360
|
+
if not isinstance(value, str):
|
|
361
|
+
raise ValueError("Invalid field name.")
|
|
362
|
+
return value
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _set(target: JsonObject, key: str, value: JsonValue) -> None:
|
|
366
|
+
if key in target:
|
|
367
|
+
raise ValueError("Duplicate key " + key + ".")
|
|
368
|
+
target[key] = value
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _set_path(target: JsonObject, path: list[str], value: JsonValue) -> None:
|
|
372
|
+
if not path:
|
|
373
|
+
raise ValueError("Empty column path.")
|
|
374
|
+
for index, key in enumerate(path):
|
|
375
|
+
if index == len(path) - 1:
|
|
376
|
+
_set(target, key, value)
|
|
377
|
+
return
|
|
378
|
+
if key not in target:
|
|
379
|
+
_set(target, key, {})
|
|
380
|
+
nested = target[key]
|
|
381
|
+
if not isinstance(nested, dict):
|
|
382
|
+
raise ValueError("Conflicting column paths.")
|
|
383
|
+
target = nested
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _read_table(lines: list[str], start: int, count: int, keyed: bool, row_depth: int, features: set[str] | None = None) -> JsonRoot:
|
|
387
|
+
if start >= len(lines):
|
|
388
|
+
raise ValueError("Missing table header.")
|
|
389
|
+
fields = lines[start].split("\t")
|
|
390
|
+
if keyed and fields.pop(0) != "@key":
|
|
391
|
+
raise ValueError("Missing @key marker.")
|
|
392
|
+
if not fields:
|
|
393
|
+
raise ValueError("Missing table fields.")
|
|
394
|
+
paths: list[list[str]] = []
|
|
395
|
+
for field in fields:
|
|
396
|
+
path = [_read_key(field)] if field.startswith('"') else field.split(".", _MAX_HEADER_SEGMENTS)
|
|
397
|
+
if len(path) > _MAX_HEADER_SEGMENTS:
|
|
398
|
+
raise ValueError("Maximum header path is 8 segments.")
|
|
399
|
+
if row_depth + len(path) > _MAX_JSON_DEPTH:
|
|
400
|
+
raise ValueError(_DEPTH_ERROR)
|
|
401
|
+
paths.append(path)
|
|
402
|
+
if features is not None:
|
|
403
|
+
if keyed:
|
|
404
|
+
features.add("keyed")
|
|
405
|
+
if any(len(path) > 1 for path in paths):
|
|
406
|
+
features.add("nested")
|
|
407
|
+
trie: _HeaderTrie = {}
|
|
408
|
+
for index, path in enumerate(paths):
|
|
409
|
+
if not fields[index].startswith('"') and not all(_segment(part) for part in path):
|
|
410
|
+
raise ValueError("Invalid header.")
|
|
411
|
+
node = trie
|
|
412
|
+
for position, segment in enumerate(path):
|
|
413
|
+
if position == len(path) - 1:
|
|
414
|
+
if segment in node:
|
|
415
|
+
raise ValueError("Duplicate or conflicting header.")
|
|
416
|
+
node[segment] = None
|
|
417
|
+
elif segment in node:
|
|
418
|
+
child = node[segment]
|
|
419
|
+
if child is None:
|
|
420
|
+
raise ValueError("Duplicate or conflicting header.")
|
|
421
|
+
node = child
|
|
422
|
+
else:
|
|
423
|
+
child = {}
|
|
424
|
+
node[segment] = child
|
|
425
|
+
node = child
|
|
426
|
+
ids = not keyed and fields[0] == "id"
|
|
427
|
+
result: JsonRoot = {} if keyed else []
|
|
428
|
+
for index in range(count):
|
|
429
|
+
if start + 1 + index >= len(lines):
|
|
430
|
+
raise ValueError("Row count mismatch.")
|
|
431
|
+
line = lines[start + 1 + index]
|
|
432
|
+
cells = line.split("\t")
|
|
433
|
+
if len(cells) != len(fields) + int(keyed):
|
|
434
|
+
raise ValueError("Wrong cell count.")
|
|
435
|
+
entry_key = cells.pop(0) if keyed else None
|
|
436
|
+
row: JsonObject = {}
|
|
437
|
+
for column, (path, raw) in enumerate(zip(paths, cells)):
|
|
438
|
+
if column == 0 and ids and not raw:
|
|
439
|
+
raise ValueError("Missing string id.")
|
|
440
|
+
if raw:
|
|
441
|
+
_set_path(row, path, raw if column == 0 and ids else _read_cell(raw))
|
|
442
|
+
if isinstance(result, list):
|
|
443
|
+
result.append(row)
|
|
444
|
+
else:
|
|
445
|
+
if entry_key is None or not _safe_id(entry_key) or _ENTRY.fullmatch(entry_key):
|
|
446
|
+
raise ValueError("Invalid entry key.")
|
|
447
|
+
_set(result, entry_key, row)
|
|
448
|
+
return result
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _read_count(text: str) -> int:
|
|
452
|
+
# Bound the digit length before int conversion, including adversarial input.
|
|
453
|
+
if len(text) > 16 or not text.isascii() or not text.isdigit():
|
|
454
|
+
raise ValueError("Invalid table count.")
|
|
455
|
+
count = int(text)
|
|
456
|
+
if count < 2 or count > 9007199254740991:
|
|
457
|
+
raise ValueError("Invalid table count.")
|
|
458
|
+
return count
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def decode(text: str) -> JsonRoot:
|
|
462
|
+
"""Decode canonical text. Invalid text raises ValueError.
|
|
463
|
+
|
|
464
|
+
Integer tokens within [-2**53, 2**53] become ints. Other numbers are floats.
|
|
465
|
+
Wrong argument types raise TypeError.
|
|
466
|
+
"""
|
|
467
|
+
return _read_document(text)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _read_document(text: str, features: set[str] | None = None) -> JsonRoot:
|
|
471
|
+
if not isinstance(text, str):
|
|
472
|
+
raise TypeError("Expected Terset text.")
|
|
473
|
+
lines = text.split("\n")
|
|
474
|
+
root_count = re.fullmatch(r"\[([0-9]+)\]", lines[0]) if len(lines) > 1 else None
|
|
475
|
+
result: JsonRoot
|
|
476
|
+
if root_count:
|
|
477
|
+
count = _read_count(root_count[1])
|
|
478
|
+
result = _read_table(lines, 1, count, False, 1, features)
|
|
479
|
+
if len(lines) != count + 2:
|
|
480
|
+
raise ValueError("Row count mismatch.")
|
|
481
|
+
elif text == "{}":
|
|
482
|
+
result = {}
|
|
483
|
+
elif text.startswith("["):
|
|
484
|
+
parsed = _parse(text)
|
|
485
|
+
if not isinstance(parsed, list):
|
|
486
|
+
raise ValueError("Expected a JSON array.")
|
|
487
|
+
result = parsed
|
|
488
|
+
else:
|
|
489
|
+
result = {}
|
|
490
|
+
index = 0
|
|
491
|
+
while index < len(lines):
|
|
492
|
+
match = _ENTRY.fullmatch(lines[index])
|
|
493
|
+
index += 1
|
|
494
|
+
if match is None:
|
|
495
|
+
raise ValueError("Expected root field.")
|
|
496
|
+
key = _read_key(match[1])
|
|
497
|
+
rest = match[4]
|
|
498
|
+
if match[2] is not None or match[3] is not None:
|
|
499
|
+
if rest != "":
|
|
500
|
+
raise ValueError("Expected a table header.")
|
|
501
|
+
count = _read_count(match[2] if match[2] is not None else match[3])
|
|
502
|
+
_set(result, key, _read_table(lines, index, count, match[3] is not None, 2, features))
|
|
503
|
+
index += count + 1
|
|
504
|
+
else:
|
|
505
|
+
if not rest.startswith(" "):
|
|
506
|
+
raise ValueError("Expected scalar field.")
|
|
507
|
+
_set(result, key, _read_cell(rest[1:]))
|
|
508
|
+
try:
|
|
509
|
+
canonical = encode(result)
|
|
510
|
+
except (TypeError, RecursionError) as error:
|
|
511
|
+
raise ValueError(str(error)) from error
|
|
512
|
+
if canonical != text:
|
|
513
|
+
raise ValueError("Noncanonical Terset text.")
|
|
514
|
+
return result
|
terset/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: terset
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Encode JSON records as compact, typed tables.
|
|
5
|
+
Author: TrueWear, LLC
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://terset.dev
|
|
8
|
+
Project-URL: Source, https://github.com/TrueWear/terset
|
|
9
|
+
Keywords: json,llm,prompt,tokens,table,encoding
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# Terset for Python
|
|
25
|
+
|
|
26
|
+
Terset writes JSON records as compact, typed tables using format v2. Python 3.10 or newer.
|
|
27
|
+
Zero runtime dependencies.
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
Coming soon on PyPI. This package is not published yet.
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
pip install terset
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Use
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from terset import decode, encode, rules_for
|
|
41
|
+
|
|
42
|
+
data = [{"id": "1", "name": "Ada"}, {"id": "2", "name": "Ben"}]
|
|
43
|
+
text = encode(data)
|
|
44
|
+
print(text)
|
|
45
|
+
print(decode(text) == data)
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Output (the gaps are tabs):
|
|
49
|
+
|
|
50
|
+
```text
|
|
51
|
+
[2]
|
|
52
|
+
id name
|
|
53
|
+
1 Ada
|
|
54
|
+
2 Ben
|
|
55
|
+
True
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
`encode(value)` accepts a dictionary or list at the root. Values can be
|
|
59
|
+
strings, finite numbers, booleans, `None`, dictionaries with string keys,
|
|
60
|
+
and lists. It returns canonical Terset text. Put `rules_for(text)` before the data
|
|
61
|
+
when a reader needs the format explained. It validates canonical text and returns
|
|
62
|
+
the base rules, plus rules for keyed tables and dotted headers when present.
|
|
63
|
+
The exported `RULES` keeps all rules for any text. `decode(text)` returns the same
|
|
64
|
+
JSON value and rejects text the encoder would not produce.
|
|
65
|
+
|
|
66
|
+
Lists of at least two dictionaries become tables when the header and rows
|
|
67
|
+
save UTF-8 bytes. Named tables start with `name[N]:`. Root tables start
|
|
68
|
+
with `[N]` on its own line. Compact JSON lists have no count.
|
|
69
|
+
Missing fields, empty strings and `None` stay distinct.
|
|
70
|
+
|
|
71
|
+
A map in a direct root field can use `name{N}:` when the complete named
|
|
72
|
+
block is strictly shorter than `name: JSON`. Its first column is `@key`,
|
|
73
|
+
which holds each entry's key and is not a field. Root maps stay fields.
|
|
74
|
+
Uniform nested objects with identical ordered simple keys expand into
|
|
75
|
+
dotted headers, recursively. Literal dotted field names are quoted.
|
|
76
|
+
Other nested values stay compact JSON. The decoder rejects count mismatches
|
|
77
|
+
and accepts only the current canonical format.
|
|
78
|
+
See the full [format v2 specification](https://github.com/TrueWear/terset/blob/main/packages/terset/SPEC.md).
|
|
79
|
+
|
|
80
|
+
## Python details
|
|
81
|
+
|
|
82
|
+
- The format uses JavaScript binary64 numbers. `1.0` writes as `1`.
|
|
83
|
+
Decoded integer literals within `[-2**53, 2**53]` become Python `int`
|
|
84
|
+
values. Tokens with a fraction or exponent, and integers outside this
|
|
85
|
+
range, become `float` values.
|
|
86
|
+
Python integers must be exactly representable as binary64 numbers;
|
|
87
|
+
otherwise encoding raises `TypeError`. This avoids silently rounding
|
|
88
|
+
integers such as `9007199254740993`.
|
|
89
|
+
- Numeric negative zero, infinities, NaN, cycles, non-JSON types and depth
|
|
90
|
+
above 500 raise `TypeError`. Use plain built-in dictionaries and lists.
|
|
91
|
+
- Dictionary order is preserved, with JavaScript's array-index keys first
|
|
92
|
+
in numeric order. For example, `"2"` precedes `"10"`; `"01"` keeps its
|
|
93
|
+
insertion position among other keys.
|
|
94
|
+
- Unicode stays unchanged, including normalization forms and lone
|
|
95
|
+
surrogates. Explicit UTF-16 surrogate pairs become Python astral code
|
|
96
|
+
points. Keys that become identical through this conversion are rejected.
|
|
97
|
+
- Format validation errors raise `ValueError`, including non-finite numbers
|
|
98
|
+
and excessive depth. Passing a non-string to `decode` raises `TypeError`.
|
|
99
|
+
|
|
100
|
+
`encode` uses the covariant `JsonInputRoot` and `JsonInputValue` aliases,
|
|
101
|
+
based on read-only `Mapping` and `Sequence` interfaces. This lets typed
|
|
102
|
+
lists and dictionaries pass without widening their element types. Runtime
|
|
103
|
+
validation still requires plain built-in lists and dictionaries.
|
|
104
|
+
`decode` returns concrete `JsonRoot`, `JsonObject` and `JsonValue` types.
|
|
105
|
+
The package includes `py.typed`.
|
|
106
|
+
|
|
107
|
+
## Limits
|
|
108
|
+
|
|
109
|
+
The format rejects numeric negative zero and nesting deeper than 500.
|
|
110
|
+
With deep data and an already deep caller stack,
|
|
111
|
+
`encode`, `decode` and `rules_for` can raise `RecursionError`.
|
|
112
|
+
This depends on the available stack depth, not the Python version.
|
|
113
|
+
|
|
114
|
+
## Local tests
|
|
115
|
+
|
|
116
|
+
From the repository root, without installing the package:
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
python3 -B -m unittest discover -s packages/terset-py/tests -v
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Tests read the shared cases generated by the TypeScript encoder and check
|
|
123
|
+
exact output, decoding, the rules string, and invalid inputs. To regenerate
|
|
124
|
+
the cases from the repository root with Node 24:
|
|
125
|
+
|
|
126
|
+
```sh
|
|
127
|
+
node packages/terset/tests/generate-cases.ts
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
## License
|
|
131
|
+
|
|
132
|
+
MIT. Copyright (c) 2026 TrueWear, LLC. The license text is included in `LICENSE`.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
terset/__init__.py,sha256=lkRAXYSzfizQ94jdIl6eoz5j1QLyRj7QwA1rgjXQd6E,323
|
|
2
|
+
terset/_terset.py,sha256=NIeUmbGAotroME4jb_WmBS1XLf32RP9Uejyh3orxxIE,21005
|
|
3
|
+
terset/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
terset-0.2.1.dist-info/licenses/LICENSE,sha256=KFZPha3jIByEKJi_aDtHArSXpKTJRHWRqyct5pihGAg,1070
|
|
5
|
+
terset-0.2.1.dist-info/METADATA,sha256=6nGD06x4vWDyJcMmsgmb5sYF2pUQhPd4M4fEKTvb878,5020
|
|
6
|
+
terset-0.2.1.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
7
|
+
terset-0.2.1.dist-info/top_level.txt,sha256=lr1EnjARmx3cWvS6kuRF5-wzEEsCVAMVjhqirfF-YmM,7
|
|
8
|
+
terset-0.2.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 TrueWear, LLC
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
terset
|