json-compress 1.0.3__py3-none-any.whl

json_compress/__init__.py

@@ -0,0 +1,242 @@
+"""json-compress: lossless JSON compression that is still JSON.
+
+Three ideas, applied together:
+
+1. Every object key is replaced by a short token and the real names are listed
+   once, in a catalogue. A key that occurs ten thousand times is spelled out
+   once.
+2. A list of like-shaped dicts becomes a table - one column list and a matrix of
+   rows - so the keys disappear from the rows entirely. Columns whose value
+   never changes are hoisted out of the rows altogether, and columns drawn from
+   a small set of strings become integer indices.
+3. String values that repeat are moved into a second catalogue and referenced by
+   token.
+
+The output is an ordinary JSON value with no binary framing, so it survives
+anything that carries JSON: an HTTP body, a WebSocket frame, a ``jsonb`` column,
+a queue message, a log line.
+
+    >>> from json_compress import compress, decompress
+    >>> envelope = compress([{"id": 1, "role": "admin"}, {"id": 2, "role": "admin"}])
+    >>> decompress(envelope)
+    [{'id': 1, 'role': 'admin'}, {'id': 2, 'role': 'admin'}]
+
+The wire format is specified in ``SPEC.md`` at the root of the repository, and
+this implementation writes the same bytes as the JavaScript and .NET ones.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Mapping, Optional, Union
+
+from .constants import FORMAT_VERSION
+from .decode import DecodeError, decompress, is_compressed
+from .encode import compress
+from .jsontext import dumps, loads
+from .locate import JsonLocation, locate
+from .normalize import normalize
+from .options import DEFAULT_OPTIONS, CompressOptions, resolve_options
+from .pack import is_packed, pack, pack_all, text_to_array, unpack, unpack_all
+from .sizes import byte_length, gunzip, gzip_bytes, gzip_size
+
+__version__ = "1.0.3"
+
+__all__ = [
+    "FORMAT_VERSION",
+    "__version__",
+    "CompressOptions",
+    "DEFAULT_OPTIONS",
+    "DecodeError",
+    "JsonCompressCodec",
+    "JsonLocation",
+    "SizeReport",
+    "beautify",
+    "byte_length",
+    "compress",
+    "create_codec",
+    "decompress",
+    "decompress_if_needed",
+    "dumps",
+    "gunzip",
+    "gzip_bytes",
+    "gzip_size",
+    "is_compressed",
+    "is_packed",
+    "loads",
+    "locate",
+    "measure",
+    "minify",
+    "pack",
+    "pack_all",
+    "parse",
+    "stringify",
+    "text_to_array",
+    "unpack",
+    "unpack_all",
+    "validate",
+]
+
+_Options = Union[CompressOptions, Mapping[str, Any], None]
+
+
+def stringify(
+    value: Any,
+    options: _Options = None,
+    indent: Union[int, str, None] = None,
+    **overrides: Any,
+) -> str:
+    """Compress a value and serialise it in one step."""
+    return dumps(compress(value, options, **overrides), indent)
+
+
+def parse(text: str) -> Any:
+    """Parse a compressed document and restore the original value."""
+    return decompress(loads(text))
+
+
+def decompress_if_needed(value: Any) -> Any:
+    """Restore a value whether or not it was compressed.
+
+    Handy at a boundary you do not fully control - an endpoint being migrated, a
+    cache holding entries written by two versions of the same service.
+    """
+    return decompress(value) if is_compressed(value) else value
+
+
+def minify(text: str) -> str:
+    """Strip the whitespace from a JSON document without changing what it means."""
+    return dumps(loads(text))
+
+
+def beautify(text: str, indent: Union[int, str] = 2) -> str:
+    """Re-indent a JSON document."""
+    return dumps(loads(text), indent)
+
+
+def validate(text: str) -> JsonLocation:
+    """Check that a string is valid JSON, and say exactly where it stops being so.
+
+    The verdict comes from the standard library, so it is the same verdict
+    ``json.loads`` would give. The location comes from a scan of the document,
+    because parser messages are written for parser authors and often name the
+    wrong culprit.
+    """
+    try:
+        loads(text)
+        return JsonLocation(True)
+    except Exception as error:  # noqa: BLE001 - any parse failure is a failure
+        found = locate(text)
+        if not found.valid:
+            return found
+
+        # The scan found nothing but the parse still failed - report what the
+        # parser said rather than claiming the document is fine.
+        return JsonLocation(False, str(error))
+
+
+@dataclass(frozen=True)
+class SizeReport:
+    """What compression bought on one value."""
+
+    #: Bytes of the input as given.
+    original: int
+    #: Bytes once whitespace is removed.
+    minified: int
+    #: Bytes of the compressed envelope.
+    compressed: int
+    #: Bytes saved against the minified form.
+    saved: int
+    #: Compressed divided by minified, 0 to 1.
+    ratio: float
+    #: Percentage saved against the minified form.
+    percent: float
+    #: Minified bytes after gzip, or ``None`` when not measured.
+    gzip_minified: Optional[int] = None
+    #: Compressed bytes after gzip, or ``None`` when not measured.
+    gzip_compressed: Optional[int] = None
+
+
+def measure(
+    value: Any,
+    options: _Options = None,
+    source: Optional[str] = None,
+    gzip: bool = True,
+    **overrides: Any,
+) -> SizeReport:
+    """Measure what compression buys on a given value.
+
+    The honest comparison is against minified JSON, not against whatever
+    whitespace the input happened to arrive with, so ``ratio`` and ``percent``
+    both use the minified size as their baseline. Gzip figures are included
+    because that is what most payloads meet in production, and the interesting
+    question is whether the format still helps once gzip has had its turn.
+
+    Pass ``source`` to have ``original`` reflect the text the document really
+    arrived as, and ``gzip=False`` to skip the gzip measurements.
+    """
+    minified = dumps(normalize(value))
+    compressed = dumps(compress(value, options, **overrides))
+
+    minified_bytes = byte_length(minified)
+    compressed_bytes = byte_length(compressed)
+
+    return SizeReport(
+        original=minified_bytes if source is None else byte_length(source),
+        minified=minified_bytes,
+        compressed=compressed_bytes,
+        saved=minified_bytes - compressed_bytes,
+        ratio=1.0 if minified_bytes == 0 else compressed_bytes / minified_bytes,
+        percent=(
+            0.0
+            if minified_bytes == 0
+            else (minified_bytes - compressed_bytes) / minified_bytes * 100
+        ),
+        gzip_minified=gzip_size(minified) if gzip else None,
+        gzip_compressed=gzip_size(compressed) if gzip else None,
+    )
+
+
+class JsonCompressCodec:
+    """A set of options, bound once and reused.
+
+    A codec is the natural unit when the same options apply everywhere - a
+    transport layer, a cache adapter, a storage helper - and it keeps the option
+    object off every call site. It holds nothing but its options, which are
+    frozen, so one instance is safe to share across threads.
+    """
+
+    __slots__ = ("options",)
+
+    def __init__(self, options: _Options = None, **overrides: Any) -> None:
+        self.options = resolve_options(options, overrides)
+
+    def compress(self, value: Any) -> dict:
+        """Compress a value with this codec's options."""
+        return compress(value, self.options)
+
+    def decompress(self, envelope: Any) -> Any:
+        """Restore a compressed value."""
+        return decompress(envelope)
+
+    def stringify(self, value: Any, indent: Union[int, str, None] = None) -> str:
+        """Compress a value and serialise it."""
+        return dumps(compress(value, self.options), indent)
+
+    def parse(self, text: str) -> Any:
+        """Parse and restore a compressed document."""
+        return decompress(loads(text))
+
+    def measure(
+        self, value: Any, source: Optional[str] = None, gzip: bool = True
+    ) -> SizeReport:
+        """Measure what this codec's options buy on a value."""
+        return measure(value, self.options, source=source, gzip=gzip)
+
+    def __repr__(self) -> str:
+        return "JsonCompressCodec(" + repr(self.options) + ")"
+
+
+def create_codec(options: _Options = None, **overrides: Any) -> JsonCompressCodec:
+    """Bind a set of options once and reuse them."""
+    return JsonCompressCodec(options, **overrides)

json_compress/__main__.py

@@ -0,0 +1,5 @@
+"""Allow `python -m json_compress` to run the command line."""
+
+from .cli import main
+
+raise SystemExit(main())

json_compress/cli.py

@@ -0,0 +1,243 @@
+"""The json-compress command line.
+
+Reads from a file or from stdin, writes to a file or to stdout, so it drops into
+a pipeline without ceremony::
+
+    curl -s https://api.example.com/orders | json-compress compress > orders.jc.json
+    json-compress stats orders.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import os
+import sys
+from typing import Any, List, NoReturn, Optional
+
+from . import (
+    __version__,
+    beautify,
+    compress,
+    decompress,
+    dumps,
+    is_compressed,
+    loads,
+    measure,
+    minify,
+    validate,
+)
+
+COMMANDS = ("compress", "decompress", "stats", "minify", "beautify", "validate")
+
+_EPILOG = """
+commands:
+  compress [file]     Compress a JSON document
+  decompress [file]   Restore a compressed document
+  stats [file]        Report what compression would save, and change nothing
+  minify [file]       Remove whitespace only
+  beautify [file]     Re-indent
+  validate [file]     Check the document parses, and say where it does not
+
+With no input file, the document is read from stdin.
+"""
+
+
+def _fail(message: str) -> NoReturn:
+    sys.stderr.write("json-compress: " + message + "\n")
+    raise SystemExit(1)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="json-compress",
+        description="lossless JSON compression that is still JSON",
+        epilog=_EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("command", nargs="?", choices=COMMANDS, help=argparse.SUPPRESS)
+    parser.add_argument("input", nargs="?", help="the document to read (default stdin)")
+    parser.add_argument(
+        "-o", "--out", metavar="FILE", help="write to a file instead of stdout"
+    )
+    parser.add_argument(
+        "-p", "--pretty", action="store_true", help="indent the output"
+    )
+    parser.add_argument(
+        "-i",
+        "--indent",
+        type=int,
+        default=2,
+        metavar="N",
+        help="indent width for --pretty and beautify (default 2)",
+    )
+    parser.add_argument(
+        "--no-strings",
+        dest="strings",
+        action="store_false",
+        help="do not build a string-value catalogue",
+    )
+    parser.add_argument(
+        "--no-tables",
+        dest="tables",
+        action="store_false",
+        help="do not pack arrays of objects into tables",
+    )
+    parser.add_argument(
+        "--no-constants",
+        dest="constants",
+        action="store_false",
+        help="do not hoist unchanging columns out of rows",
+    )
+    parser.add_argument(
+        "--no-dict-columns",
+        dest="dict_columns",
+        action="store_false",
+        help="do not encode low-cardinality string columns as indices",
+    )
+    parser.add_argument(
+        "--min-rows",
+        type=int,
+        default=2,
+        metavar="N",
+        help="smallest array that may become a table (default 2)",
+    )
+    parser.add_argument(
+        "-v", "--version", action="version", version=__version__
+    )
+    return parser
+
+
+def _read_input(path: Optional[str]) -> str:
+    if path:
+        try:
+            with open(path, encoding="utf-8") as handle:
+                return handle.read()
+        except OSError as error:
+            _fail("cannot read " + path + " (" + (error.strerror or "failed") + ")")
+    if sys.stdin is None or sys.stdin.isatty():
+        _fail("no input file given and nothing on stdin")
+    return sys.stdin.read()
+
+
+def _write_output(text: str, path: Optional[str]) -> None:
+    if path:
+        with open(path, "w", encoding="utf-8", newline="\n") as handle:
+            handle.write(text)
+        return
+    sys.stdout.write(text + "\n")
+
+
+def _kilobytes(count: int) -> str:
+    if count < 1024:
+        return str(count) + " B"
+    if count < 1024 * 1024:
+        return "%.1f KB" % (count / 1024)
+    return "%.2f MB" % (count / (1024 * 1024))
+
+
+def _change(percent: float) -> str:
+    """"31% smaller" or "31% larger" - never a negative percentage."""
+    return "%.1f%% %s" % (abs(percent), "larger" if percent < 0 else "smaller")
+
+
+def _stats(report: Any, name: str) -> str:
+    lines = [
+        name,
+        "  as given          " + _kilobytes(report.original).rjust(10),
+        "  minified          " + _kilobytes(report.minified).rjust(10),
+        "  compressed        " + _kilobytes(report.compressed).rjust(10),
+        "  saved             "
+        + _kilobytes(report.saved).rjust(10)
+        + "   "
+        + _change(report.percent)
+        + " than minified",
+    ]
+    if report.gzip_minified is not None:
+        saved = report.gzip_minified - report.gzip_compressed
+        percent = 0.0 if report.gzip_minified == 0 else saved / report.gzip_minified * 100
+        lines += [
+            "",
+            "  minified + gzip   " + _kilobytes(report.gzip_minified).rjust(10),
+            "  compressed + gzip " + _kilobytes(report.gzip_compressed).rjust(10),
+            "  saved             "
+            + _kilobytes(saved).rjust(10)
+            + "   "
+            + _change(percent)
+            + " than minified + gzip",
+        ]
+    return "\n".join(lines)
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    """Run the command line. Returns the process exit code."""
+    for stream in (sys.stdin, sys.stdout, sys.stderr):
+        # A JSON document is UTF-8 whatever the console code page says, and the
+        # output is compared byte for byte by the interoperability checks.
+        if hasattr(stream, "reconfigure"):
+            # An exotic console may refuse; the default encoding then stands.
+            with contextlib.suppress(ValueError, OSError):
+                stream.reconfigure(encoding="utf-8", newline="\n")
+
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command is None:
+        parser.print_help()
+        return 0
+
+    text = _read_input(args.input)
+    indent = args.indent if args.pretty else None
+
+    if args.command == "validate":
+        found = validate(text)
+        if found.valid:
+            sys.stdout.write("valid JSON\n")
+            return 0
+        where = (
+            " at line " + str(found.line) + ", column " + str(found.column)
+            if found.line
+            else ""
+        )
+        _fail("invalid JSON" + where + ": " + str(found.error))
+
+    if args.command == "minify":
+        _write_output(minify(text), args.out)
+        return 0
+
+    if args.command == "beautify":
+        _write_output(beautify(text, args.indent), args.out)
+        return 0
+
+    try:
+        parsed = loads(text)
+    except ValueError as error:
+        _fail("input is not valid JSON: " + str(error))
+
+    options = {
+        "strings": args.strings,
+        "tables": args.tables,
+        "constants": args.constants,
+        "dict_columns": args.dict_columns,
+        "min_rows": args.min_rows,
+    }
+
+    if args.command == "compress":
+        _write_output(dumps(compress(parsed, options), indent), args.out)
+        return 0
+
+    if args.command == "decompress":
+        if not is_compressed(parsed):
+            _fail("input is not a compressed document")
+        _write_output(dumps(decompress(parsed), indent), args.out)
+        return 0
+
+    # args.command == "stats"
+    report = measure(parsed, options, source=text)
+    name = os.path.basename(args.input) if args.input else "stdin"
+    sys.stdout.write(_stats(report, name) + "\n")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

json_compress/column_order.py

@@ -0,0 +1,82 @@
+"""Choosing the column order for a table.
+
+A table has one column list, but the rows that feed it each have their own key
+order. The decoder rebuilds every row by walking the columns, so the order it
+produces is only faithful if each row's keys appear in the column list in the
+same sequence they appeared in the row.
+
+That is a topological sort. Every row contributes the constraint "this key came
+before that one", and any linear extension of those constraints reproduces every
+row exactly. Where the constraints contradict each other - one row says name
+before id, another says id before name - no column order can satisfy both, and
+the array is left as ordinary objects rather than quietly reordered.
+
+Ties are broken by first appearance, so the same input always produces the same
+output.
+"""
+
+from __future__ import annotations
+
+import heapq
+from typing import Any, Dict, List, Mapping, Optional, Sequence
+
+__all__ = ["order_columns"]
+
+
+def order_columns(rows: Sequence[Mapping[str, Any]]) -> Optional[List[str]]:
+    """Work out a column order that reproduces every row's key order.
+
+    Returns the columns, or ``None`` when no order can satisfy every row.
+    """
+    position: Dict[str, int] = {}
+    columns: List[str] = []
+
+    # First pass: collect the keys, in the order the document first mentions
+    # them. That order is the tie-break, and gives a stable result.
+    for row in rows:
+        for key in row:
+            if key not in position:
+                position[key] = len(columns)
+                columns.append(key)
+
+    count = len(columns)
+    if count <= 1:
+        return columns
+
+    after: List[set] = [set() for _ in range(count)]
+    incoming = [0] * count
+
+    # Second pass: one edge per adjacent pair of keys in each row. Transitivity
+    # along the chain covers the rest of the row's ordering.
+    saw_edge = False
+    for row in rows:
+        previous = -1
+        for key in row:
+            current = position[key]
+            if previous != -1 and current not in after[previous]:
+                after[previous].add(current)
+                incoming[current] += 1
+                saw_edge = True
+            previous = current
+
+    # Every row was a single key or empty: nothing constrains the order.
+    if not saw_edge:
+        return columns
+
+    # A heap rather than a queue, so ties break towards the key the document
+    # mentioned first however many columns there are.
+    ready = [at for at in range(count) if incoming[at] == 0]
+    heapq.heapify(ready)
+
+    ordered: List[str] = []
+    while ready:
+        at = heapq.heappop(ready)
+        ordered.append(columns[at])
+        for following in after[at]:
+            incoming[following] -= 1
+            if incoming[following] == 0:
+                heapq.heappush(ready, following)
+
+    # Fewer keys placed than we started with means the constraints form a cycle:
+    # two rows disagree about which key comes first.
+    return ordered if len(ordered) == count else None

json_compress/constants.py

@@ -0,0 +1,50 @@
+"""Wire-format constants for the json-compress envelope.
+
+Everything the encoder emits is still ordinary JSON. The format therefore has
+to be self-describing without any out-of-band type tags, which it achieves by
+reserving a single character - the tilde - inside object keys and strings.
+
+Because every original object key is replaced by a catalogue token, and key
+tokens are drawn from ``[A-Za-z][0-9A-Za-z]*``, no original key can ever appear
+as an object key in the encoded document. That makes every key that starts with
+``~`` safe to reserve for structural nodes.
+
+String values are escaped on the way in (a value that already starts with a
+tilde gains one more), which frees the ``~``-prefixed string space for catalogue
+references and the row hole marker.
+"""
+
+#: Envelope tag and format version. Bumped only on a breaking format change.
+FORMAT_VERSION = 1
+
+#: Envelope field: format version.
+ENV_VERSION = "jc"
+#: Envelope field: key catalogue, a list of the original key names.
+ENV_KEYS = "k"
+#: Envelope field: string-value catalogue. Omitted when empty.
+ENV_STRINGS = "s"
+#: Envelope field: the encoded document.
+ENV_DATA = "d"
+
+#: Table node: column list, as indices into the key catalogue.
+T_COLUMNS = "~t"
+#: Table node: per-column encoding. Omitted when every column is raw.
+T_ENCODINGS = "~e"
+#: Table node: values of the constant columns, in column order.
+T_CONSTS = "~cv"
+#: Table node: the rows, each a list of cells for the non-constant columns.
+T_ROWS = "~r"
+#: Table node: row count. Only present when every column is constant.
+T_ROWCOUNT = "~n"
+
+#: Cell marker for "this row does not have this key at all".
+HOLE = "~"
+#: Dictionary-column equivalent of :data:`HOLE`.
+DICT_HOLE = -1
+
+#: Column encoding: cells are ordinary encoded values.
+COL_RAW = 0
+#: Column encoding: cells are integer indices into the string catalogue.
+COL_DICT = 1
+#: Column encoding: the column is constant; its value lives in ``~cv``.
+COL_CONST = 2