json-razor 0.1.0__tar.gz

json_razor-0.1.0/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: json-razor
+Version: 0.1.0
+Summary: Reduces JSON, YAML, and NDJSON volume by collapsing repeated structures while preserving the schema.
+License: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: PyYAML
+
+# JSON's Razor — Cut the fat
+
+[![tests](https://github.com/rick-does/json-razor/actions/workflows/tests.yml/badge.svg)](https://github.com/rick-does/json-razor/actions/workflows/tests.yml)
+
+Reduces JSON, YAML, and NDJSON volume by collapsing repeated structures while preserving the schema.
+
+Large structured data files are hard to parse — not because the structure is complex, but because repetition obscures it. A list of 10,000 objects with identical shape tells you nothing more than a list of 1. JSON's Razor collapses that repetition to its minimum essential form: one representative example of each repeated structure, at every level of nesting.
+
+The output is valid, parseable data in the same format as input — not a summary, not a schema definition. It just has far less volume.
+
+---
+
+## Install
+
+```bash
+pip install json-razor
+```
+
+---
+
+## Usage
+
+```bash
+cat big.json | json-razor                    # stdin → stdout
+json-razor big.json                          # file input → stdout
+json-razor big.json -o small.json            # file input → file output
+json-razor big.yaml                          # auto-detected as YAML
+json-razor app.log --format ndjson           # NDJSON log file
+```
+
+### Options
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--keep N` | 1 | Number of examples to keep per repeated structure |
+| `--depth N` | unlimited | Stop collapsing below this nesting depth |
+| `--format` | auto | Force format: `json`, `yaml`, or `ndjson` |
+| `--truncate N` | 100 | Max string length before truncating |
+
+---
+
+## How it works
+
+**Arrays** — collapsed to one item. Mixed-type arrays keep one of each distinct type.
+
+```json
+// input
+[{"id": 1, "name": "alice"}, {"id": 2, "name": "bob"}, {"id": 3, "name": "carol"}]
+
+// output
+[{"id": 1, "name": "alice"}]
+```
+
+**Mixed types** — one representative per JSON type (null, bool, number, string, array, object).
+
+```json
+// input
+[1, "hello", {"id": 1}, null, true, [1, 2, 3]]
+
+// output
+[1, "hello", {"id": 1}, null, true, [1]]
+```
+
+**Nested structures** — collapsed recursively at every level.
+
+**NDJSON** — collapsed across lines; one representative line kept.
+
+**Nulls and empty values** — always preserved (`null`, `[]`, `{}`).
+
+**Long strings** — truncated to a configurable preview.
+
+---
+
+## Supported formats
+
+| Format | Auto-detected from |
+|--------|--------------------|
+| JSON | `.json` |
+| YAML | `.yaml`, `.yml` |
+| NDJSON | `.ndjson` |

json_razor-0.1.0/README.md

@@ -0,0 +1,80 @@
+# JSON's Razor — Cut the fat
+
+[![tests](https://github.com/rick-does/json-razor/actions/workflows/tests.yml/badge.svg)](https://github.com/rick-does/json-razor/actions/workflows/tests.yml)
+
+Reduces JSON, YAML, and NDJSON volume by collapsing repeated structures while preserving the schema.
+
+Large structured data files are hard to parse — not because the structure is complex, but because repetition obscures it. A list of 10,000 objects with identical shape tells you nothing more than a list of 1. JSON's Razor collapses that repetition to its minimum essential form: one representative example of each repeated structure, at every level of nesting.
+
+The output is valid, parseable data in the same format as input — not a summary, not a schema definition. It just has far less volume.
+
+---
+
+## Install
+
+```bash
+pip install json-razor
+```
+
+---
+
+## Usage
+
+```bash
+cat big.json | json-razor                    # stdin → stdout
+json-razor big.json                          # file input → stdout
+json-razor big.json -o small.json            # file input → file output
+json-razor big.yaml                          # auto-detected as YAML
+json-razor app.log --format ndjson           # NDJSON log file
+```
+
+### Options
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--keep N` | 1 | Number of examples to keep per repeated structure |
+| `--depth N` | unlimited | Stop collapsing below this nesting depth |
+| `--format` | auto | Force format: `json`, `yaml`, or `ndjson` |
+| `--truncate N` | 100 | Max string length before truncating |
+
+---
+
+## How it works
+
+**Arrays** — collapsed to one item. Mixed-type arrays keep one of each distinct type.
+
+```json
+// input
+[{"id": 1, "name": "alice"}, {"id": 2, "name": "bob"}, {"id": 3, "name": "carol"}]
+
+// output
+[{"id": 1, "name": "alice"}]
+```
+
+**Mixed types** — one representative per JSON type (null, bool, number, string, array, object).
+
+```json
+// input
+[1, "hello", {"id": 1}, null, true, [1, 2, 3]]
+
+// output
+[1, "hello", {"id": 1}, null, true, [1]]
+```
+
+**Nested structures** — collapsed recursively at every level.
+
+**NDJSON** — collapsed across lines; one representative line kept.
+
+**Nulls and empty values** — always preserved (`null`, `[]`, `{}`).
+
+**Long strings** — truncated to a configurable preview.
+
+---
+
+## Supported formats
+
+| Format | Auto-detected from |
+|--------|--------------------|
+| JSON | `.json` |
+| YAML | `.yaml`, `.yml` |
+| NDJSON | `.ndjson` |

json_razor-0.1.0/json_razor/__init__.py

@@ -0,0 +1,3 @@
+from .core import collapse
+
+__all__ = ["collapse"]

json_razor-0.1.0/json_razor/cli.py

@@ -0,0 +1,88 @@
+import argparse
+import os
+import sys
+
+from .core import collapse
+from .formats import dump, dump_ndjson, load, load_ndjson
+
+
+def detect_format(path):
+    if path is None:
+        return "json"
+    ext = os.path.splitext(path)[1].lower()
+    if ext == ".json":
+        return "json"
+    if ext in (".yaml", ".yml"):
+        return "yaml"
+    if ext == ".ndjson":
+        return "ndjson"
+    return "json"
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="json-razor",
+        description="Cut the fat. Collapse repeated structures in JSON, YAML, and NDJSON.",
+    )
+    parser.add_argument("input", nargs="?", help="Input file (default: stdin)")
+    parser.add_argument("-o", "--output", help="Output file (default: stdout)")
+    parser.add_argument(
+        "--keep",
+        type=int,
+        default=1,
+        metavar="N",
+        help="Examples to keep per repeated structure (default: 1)",
+    )
+    parser.add_argument(
+        "--depth",
+        type=int,
+        default=None,
+        metavar="N",
+        help="Stop collapsing below this nesting depth",
+    )
+    parser.add_argument(
+        "--format",
+        choices=["json", "yaml", "ndjson"],
+        help="Input format (default: auto-detect from extension)",
+    )
+    parser.add_argument(
+        "--truncate",
+        type=int,
+        default=100,
+        metavar="N",
+        help="Max string length before truncating (default: 100)",
+    )
+    args = parser.parse_args()
+
+    if args.input is None and sys.stdin.isatty():
+        parser.print_help()
+        sys.exit(0)
+
+    fmt = args.format or detect_format(args.input)
+
+    if args.input:
+        with open(args.input, "r", encoding="utf-8") as f:
+            text = f.read()
+    else:
+        text = sys.stdin.read()
+
+    if fmt == "ndjson":
+        records = load_ndjson(text)
+        collapsed = collapse(records, keep=args.keep, depth=args.depth, truncate=args.truncate)
+        result = dump_ndjson(collapsed)
+    else:
+        data = load(text, fmt)
+        collapsed = collapse(data, keep=args.keep, depth=args.depth, truncate=args.truncate)
+        result = dump(collapsed, fmt)
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(result)
+            if not result.endswith("\n"):
+                f.write("\n")
+    else:
+        print(result)
+
+
+if __name__ == "__main__":
+    main()

json_razor-0.1.0/json_razor/core.py

@@ -0,0 +1,51 @@
+def collapse(data, keep=1, depth=None, truncate=100, _depth=0):
+    if depth is not None and _depth >= depth:
+        return data
+
+    if isinstance(data, str):
+        if truncate is not None and len(data) > truncate:
+            return data[:truncate] + "..."
+        return data
+
+    if isinstance(data, bool) or data is None or isinstance(data, (int, float)):
+        return data
+
+    if isinstance(data, list):
+        if len(data) == 0:
+            return []
+
+        buckets = {}
+        for item in data:
+            key = _type_key(item)
+            if key not in buckets:
+                buckets[key] = []
+            buckets[key].append(item)
+
+        result = []
+        for items in buckets.values():
+            for item in items[:keep]:
+                result.append(collapse(item, keep, depth, truncate, _depth + 1))
+        return result
+
+    if isinstance(data, dict):
+        if len(data) == 0:
+            return {}
+        return {k: collapse(v, keep, depth, truncate, _depth + 1) for k, v in data.items()}
+
+    return data
+
+
+def _type_key(value):
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "bool"
+    if isinstance(value, (int, float)):
+        return "number"
+    if isinstance(value, str):
+        return "str"
+    if isinstance(value, list):
+        return "list"
+    if isinstance(value, dict):
+        return "dict"
+    return type(value).__name__

json_razor-0.1.0/json_razor/formats.py

@@ -0,0 +1,27 @@
+import json
+
+import yaml
+
+
+def load(text, fmt):
+    if fmt == "json":
+        return json.loads(text)
+    if fmt == "yaml":
+        return yaml.safe_load(text)
+    raise ValueError(f"Unknown format: {fmt}")
+
+
+def dump(data, fmt):
+    if fmt == "json":
+        return json.dumps(data, indent=2)
+    if fmt == "yaml":
+        return yaml.dump(data, default_flow_style=False, allow_unicode=True)
+    raise ValueError(f"Unknown format: {fmt}")
+
+
+def load_ndjson(text):
+    return [json.loads(line) for line in text.splitlines() if line.strip()]
+
+
+def dump_ndjson(records):
+    return "\n".join(json.dumps(r) for r in records)

json_razor-0.1.0/json_razor.egg-info/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: json-razor
+Version: 0.1.0
+Summary: Reduces JSON, YAML, and NDJSON volume by collapsing repeated structures while preserving the schema.
+License: MIT
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: PyYAML
+
+# JSON's Razor — Cut the fat
+
+[![tests](https://github.com/rick-does/json-razor/actions/workflows/tests.yml/badge.svg)](https://github.com/rick-does/json-razor/actions/workflows/tests.yml)
+
+Reduces JSON, YAML, and NDJSON volume by collapsing repeated structures while preserving the schema.
+
+Large structured data files are hard to parse — not because the structure is complex, but because repetition obscures it. A list of 10,000 objects with identical shape tells you nothing more than a list of 1. JSON's Razor collapses that repetition to its minimum essential form: one representative example of each repeated structure, at every level of nesting.
+
+The output is valid, parseable data in the same format as input — not a summary, not a schema definition. It just has far less volume.
+
+---
+
+## Install
+
+```bash
+pip install json-razor
+```
+
+---
+
+## Usage
+
+```bash
+cat big.json | json-razor                    # stdin → stdout
+json-razor big.json                          # file input → stdout
+json-razor big.json -o small.json            # file input → file output
+json-razor big.yaml                          # auto-detected as YAML
+json-razor app.log --format ndjson           # NDJSON log file
+```
+
+### Options
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--keep N` | 1 | Number of examples to keep per repeated structure |
+| `--depth N` | unlimited | Stop collapsing below this nesting depth |
+| `--format` | auto | Force format: `json`, `yaml`, or `ndjson` |
+| `--truncate N` | 100 | Max string length before truncating |
+
+---
+
+## How it works
+
+**Arrays** — collapsed to one item. Mixed-type arrays keep one of each distinct type.
+
+```json
+// input
+[{"id": 1, "name": "alice"}, {"id": 2, "name": "bob"}, {"id": 3, "name": "carol"}]
+
+// output
+[{"id": 1, "name": "alice"}]
+```
+
+**Mixed types** — one representative per JSON type (null, bool, number, string, array, object).
+
+```json
+// input
+[1, "hello", {"id": 1}, null, true, [1, 2, 3]]
+
+// output
+[1, "hello", {"id": 1}, null, true, [1]]
+```
+
+**Nested structures** — collapsed recursively at every level.
+
+**NDJSON** — collapsed across lines; one representative line kept.
+
+**Nulls and empty values** — always preserved (`null`, `[]`, `{}`).
+
+**Long strings** — truncated to a configurable preview.
+
+---
+
+## Supported formats
+
+| Format | Auto-detected from |
+|--------|--------------------|
+| JSON | `.json` |
+| YAML | `.yaml`, `.yml` |
+| NDJSON | `.ndjson` |

json_razor-0.1.0/json_razor.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+README.md
+pyproject.toml
+json_razor/__init__.py
+json_razor/cli.py
+json_razor/core.py
+json_razor/formats.py
+json_razor.egg-info/PKG-INFO
+json_razor.egg-info/SOURCES.txt
+json_razor.egg-info/dependency_links.txt
+json_razor.egg-info/entry_points.txt
+json_razor.egg-info/requires.txt
+json_razor.egg-info/top_level.txt
+tests/test_core.py

json_razor-0.1.0/json_razor.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

json_razor-0.1.0/json_razor.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+json-razor = json_razor.cli:main

json_razor-0.1.0/json_razor.egg-info/requires.txt

@@ -0,0 +1 @@
+PyYAML

json_razor-0.1.0/json_razor.egg-info/top_level.txt

@@ -0,0 +1 @@
+json_razor

json_razor-0.1.0/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "json-razor"
+version = "0.1.0"
+description = "Reduces JSON, YAML, and NDJSON volume by collapsing repeated structures while preserving the schema."
+readme = "README.md"
+requires-python = ">=3.8"
+license = {text = "MIT"}
+dependencies = ["PyYAML"]
+
+[project.scripts]
+json-razor = "json_razor.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["json_razor*"]

json_razor-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

json_razor-0.1.0/tests/test_core.py

@@ -0,0 +1,82 @@
+import pytest
+from json_razor.core import collapse
+
+
+def test_array_collapse():
+    data = [{"id": 1}, {"id": 2}, {"id": 3}]
+    assert collapse(data) == [{"id": 1}]
+
+
+def test_array_keep_n():
+    data = [{"id": 1}, {"id": 2}, {"id": 3}]
+    assert collapse(data, keep=2) == [{"id": 1}, {"id": 2}]
+
+
+def test_mixed_type_array():
+    data = [1, "hello", {"id": 1}, None]
+    result = collapse(data)
+    types = {type(x) for x in result}
+    assert types == {int, str, dict, type(None)}
+    assert len(result) == 4
+
+
+def test_single_item_array_unchanged():
+    data = [{"id": 1}]
+    assert collapse(data) == [{"id": 1}]
+
+
+def test_empty_array_preserved():
+    assert collapse([]) == []
+
+
+def test_empty_object_preserved():
+    assert collapse({}) == {}
+
+
+def test_null_preserved():
+    assert collapse(None) is None
+
+
+def test_nested_array_collapse():
+    data = {"items": [{"a": 1}, {"a": 2}, {"a": 3}]}
+    assert collapse(data) == {"items": [{"a": 1}]}
+
+
+def test_string_truncation():
+    long = "x" * 200
+    result = collapse(long, truncate=100)
+    assert result == "x" * 100 + "..."
+
+
+def test_string_no_truncation_when_short():
+    s = "hello"
+    assert collapse(s, truncate=100) == "hello"
+
+
+def test_depth_limit():
+    data = {"a": [1, 2, 3]}
+    result = collapse(data, depth=1)
+    assert result == {"a": [1, 2, 3]}
+
+
+def test_bool_not_confused_with_int():
+    data = [True, False, 1, 0]
+    result = collapse(data)
+    types = {type(x) for x in result}
+    assert bool in types
+    assert int in types
+
+
+def test_object_keys_preserved():
+    data = {"name": "foo", "age": 30, "items": [1, 2, 3]}
+    result = collapse(data)
+    assert "name" in result
+    assert "age" in result
+    assert result["items"] == [1]
+
+
+def test_ndjson_style():
+    records = [{"id": i, "val": i * 2} for i in range(100)]
+    result = collapse(records)
+    assert len(result) == 1
+    assert result[0] == {"id": 0, "val": 0}