jsonl-diff 0.1.0__py3-none-any.whl

jsonl_diff-0.1.0.dist-info/METADATA

@@ -0,0 +1,548 @@
+Metadata-Version: 2.4
+Name: jsonl-diff
+Version: 0.1.0
+Summary: Strict, disk-backed reconciliation of large JSONL/NDJSON datasets for Python.
+Author-email: rmoralespp <rmoralespp@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Rolando Morales Perez
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Source, https://github.com/rmoralespp/jsonl-diff
+Project-URL: Homepage, https://github.com/rmoralespp/jsonl-diff
+Project-URL: Changelog, https://github.com/rmoralespp/jsonl-diff/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/rmoralespp/jsonl-diff/issues
+Project-URL: Documentation, https://rmoralespp.github.io/jsonl-diff/
+Keywords: jsonlines,ndjson,jsonl,json,gzip,bzip2,xz,zst,utilities,serialization,deserialization,diff,json-diff,cli
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Internet
+Classifier: Topic :: Utilities
+Classifier: Topic :: File Formats :: JSON
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: py-jsonl==1.5.2
+Dynamic: license-file
+
+# jsonl-diff
+
+----
+
+<p>
+  <img src="https://img.shields.io/pypi/v/jsonl-diff.svg" alt="PyPI">
+  <img src="https://img.shields.io/pypi/pyversions/jsonl-diff.svg" alt="Python">
+  <img src="https://github.com/rmoralespp/jsonl-diff/workflows/CI/badge.svg" alt="CI">
+  <img src="https://codecov.io/gh/rmoralespp/jsonl-diff/branch/main/graph/badge.svg" alt="Coverage">
+  <img src="https://img.shields.io/github/license/rmoralespp/jsonl-diff.svg" alt="License">
+</p>
+
+`jsonl-diff` performs strict, identity-based reconciliation of JSONL/NDJSON
+datasets. It matches records by one or more top-level fields, then classifies
+each identity as `equal`, `added`, `deleted`, or `modified`.
+
+## Why jsonl-diff?
+
+A text diff compares lines. That is usually the wrong model for datasets:
+reordering unchanged records creates noise, and inserting one record can make
+every later line appear different.
+
+`jsonl-diff` fills the gap between line-oriented tools and in-memory dataframe comparisons:
+
+- match records by identity, not physical position;
+- compare datasets larger than available RAM using temporary SQLite storage;
+- preserve original OLD and NEW line numbers for auditability;
+- expose the same behavior through a CLI and a small Python API.
+
+Use it for snapshots, ETL validation, migrations, exports, and CI checks. Use
+`diff`, `jq`, or a visual JSON diff instead when you need line-level,
+field-level, or JSON Patch output.
+
+The comparison is backed by a temporary SQLite index. Complete inputs and
+complete result sets do not need to fit in memory. Process memory is bounded by
+the largest record plus parser and SQLite buffers, while indexed data lives on
+temporary disk.
+
+## Features
+
+- Single or composite top-level identities, independent of input order.
+- Configurable duplicate handling with strict failure by default.
+- Exact RFC 6901 object-member ignores.
+- Semantic number comparison without binary floating-point rounding.
+- Deterministic summaries and changed-identity iteration.
+- Original OLD and NEW physical line numbers for every change.
+- Local, HTTP/HTTPS, file-like, and supported compressed sources through
+  `py-jsonl`.
+- The same comparison engine through the CLI and Python API.
+- Disk-backed comparison with configurable `jsonl-diff` temporary storage.
+
+## Requirements and installation
+
+`jsonl-diff` supports Python 3.8 through 3.14.
+
+Install it in a project managed by `uv`:
+
+```bash
+uv add jsonl-diff
+```
+
+Install the CLI as a standalone tool:
+
+```bash
+uv tool install jsonl-diff
+```
+
+`pip` is also supported:
+
+```bash
+python -m pip install jsonl-diff
+```
+
+## CLI quick start
+
+Given:
+
+```jsonl
+{"id":1,"name":"Ada"}
+{"id":2,"name":"Grace"}
+```
+
+and a reordered, changed dataset:
+
+```jsonl
+{"id":2,"name":"Grace Hopper"}
+{"name":"Ada","id":1}
+{"id":3,"name":"Linus"}
+```
+
+compare them by `id`:
+
+```bash
+jsonl-diff old.jsonl new.jsonl --key id
+```
+
+Standard output contains only the text summary:
+
+```text
+Records:
+  equal:     1
+  added:     1
+  deleted:   0
+  modified:  1
+```
+
+The result means one record is unchanged, one was added, and one existing
+record changed. Physical order does not affect these counts.
+
+Diagnostics are written to standard error. Use `--quiet` when only the exit
+status or details file is needed.
+
+### Common pipelines
+
+Filter JSONL with `jq` before comparing it. `-c` keeps one compact JSON object
+per line:
+
+```bash
+jq -c 'select(has("deleted_at") and .deleted_at == null)' old.jsonl |
+  jsonl-diff - new.jsonl --key id
+```
+
+Read compressed input directly when no preprocessing is required:
+
+```bash
+jsonl-diff old.jsonl.xz new.jsonl.gz --key id
+```
+
+For two independent pipelines on Bash-compatible systems, use process
+substitution:
+
+```bash
+jsonl-diff \
+  <(head -n 1000 old.jsonl | jq -c 'select(.active)') \
+  <(head -n 1000 new.jsonl | jq -c 'select(.active)') \
+  --key id
+```
+
+`-` represents stdin. Both inputs cannot use the same stdin stream.
+
+### CLI syntax and options
+
+```text
+jsonl-diff [-h] --key KEY [--ignore IGNORE]
+           [--duplicates {error,first,last}] [--details FILE] [--quiet]
+           [--max-temp MAX_TEMP]
+           old new
+```
+
+| Argument | Meaning |
+|---|---|
+| `old` | OLD local path, HTTP/HTTPS source, or `-` for stdin. |
+| `new` | NEW local path, HTTP/HTTPS source, or `-` for stdin. |
+| `--key KEY` | Required top-level identity field. Repeat it or use a comma-separated value for a composite identity. |
+| `--ignore JSON_POINTER` | Exact RFC 6901 pointer to an object member to remove before content comparison. Repeatable. |
+| `--duplicates POLICY` | Handle repeated identities with `error` (default), `first`, or `last`. |
+| `--details FILE` | Write machine-readable JSONL details to `FILE`; details are never written to stdout. |
+| `--quiet` | Suppress the normal stdout summary. Errors still go to stderr. |
+| `--max-temp MAX_TEMP` | Limit storage owned by `jsonl-diff` to a positive integer number of bytes. |
+| `-h`, `--help` | Show command help and exit. |
+
+These composite-key forms are equivalent:
+
+```bash
+jsonl-diff old.jsonl new.jsonl --key country,customerId,type
+
+jsonl-diff old.jsonl new.jsonl \
+  --key country \
+  --key customerId \
+  --key type
+```
+
+Ignore volatile fields by exact path:
+
+```bash
+jsonl-diff old.jsonl new.jsonl \
+  --key id \
+  --ignore /updated_at \
+  --ignore /metadata/request_id
+```
+
+### Exit codes
+
+| Code | Meaning |
+|---:|---|
+| `0` | The inputs are equal under the configured rules. |
+| `1` | At least one added, deleted, or modified identity was found. |
+| `2` | A handled input, source, resource, output, or interruption error occurred. |
+| `3` | CLI usage or comparison configuration is invalid. |
+
+Invalid argparse usage exits directly with code `3`. The Python API uses
+exceptions instead of these process exit codes.
+
+## Details JSONL
+
+`--details FILE` streams a deterministic JSONL report through `py-jsonl` after
+both inputs have been completely indexed and validated:
+
+```bash
+jsonl-diff old.jsonl new.jsonl --key id --details changes.jsonl
+```
+
+For an OLD source containing identities `1` and `2`, and a NEW source
+containing changed identity `2` and added identity `3`, the file is:
+
+```jsonl
+{"duplicates":"error","ignore":[],"key":["id"],"type":"meta"}
+{"key":[1],"old_line":1,"op":"deleted","type":"change"}
+{"key":[2],"new_line":1,"old_line":2,"op":"modified","type":"change"}
+{"key":[3],"new_line":2,"op":"added","type":"change"}
+{"added":1,"deleted":1,"equal":0,"modified":1,"type":"summary"}
+```
+
+Record types are:
+
+- **`meta`**: always first. `key` contains normalized identity-field names;
+  `ignore` contains the configured ignore pointers.
+- **`change`**: one per changed identity. `op` is `added`, `deleted`, or
+  `modified`. `old_line` is present for deletions and modifications;
+  `new_line` is present for additions and modifications. Equal records are not
+  emitted.
+- **`summary`**: always last after a successful write, with all four totals.
+
+The normal stdout summary is unchanged when `--details` is used, unless
+`--quiet` suppresses it. A write failure can leave a partial details file and
+returns exit code `2`.
+
+## Python API
+
+### Comparing sources
+
+```python
+from jsonl_diff import ChangeOperation, diff
+
+with diff(
+    "old.jsonl.gz",
+    "new.jsonl.gz",
+    key=("country", "customerId"),
+    ignore=("/updated_at",),
+    duplicates="error",
+    max_temp=2_000_000_000,
+) as result:
+    print(result.summary)
+
+    for change in result.changes(ChangeOperation.MODIFIED):
+        print(change.key, change.old_line, change.new_line)
+```
+
+The callable signature is:
+
+```python
+from typing import Any, Optional, Sequence, Union
+
+from jsonl_diff import DiffResult, DuplicatePolicy
+
+def diff(
+    old: Any,
+    new: Any,
+    *,
+    key: Union[str, Sequence[str]],
+    ignore: Sequence[str] = (),
+    duplicates: Union[str, DuplicatePolicy] = DuplicatePolicy.ERROR,
+    max_temp: Optional[int] = None,
+) -> DiffResult:
+    ...
+```
+
+`diff()` creates a comparison session without opening either source. Entering
+its context fully reads, indexes, and validates both sources before exposing
+the result. `DiffResult.changes()` is then a lazy iterator over the disk-backed
+result rather than a list held in memory.
+
+Using `DiffResult` as a context manager is required. It owns the temporary
+resources and removes its private workspace on exit. A result cannot be entered
+more than once. Its summary becomes available after entering and remains
+available after closing; iterating changes requires the context to remain open.
+
+### Result models
+
+`result.summary` is an immutable `Summary`:
+
+```python
+Summary(equal=10, added=2, deleted=1, modified=3)
+```
+
+It exposes integer fields `equal`, `added`, `deleted`, and `modified`, plus the
+boolean property `different`. The same values are available directly as
+read-only `DiffResult` properties.
+
+Each item from `result.changes()` is an immutable `Change` with:
+
+- `operation`: a `ChangeOperation` enum value (`ADDED`, `DELETED`, or
+  `MODIFIED`);
+- `key`: a typed tuple in normalized identity-field order;
+- `old` and `new`: optional `SourceLocation` values;
+- `old_line` and `new_line`: convenience properties returning a one-based
+  physical line or `None`.
+
+For additions, `old`/`old_line` are `None`; for deletions,
+`new`/`new_line` are `None`; modifications have both locations. Location
+sources are labeled `OLD` and `NEW`.
+
+Pass a `ChangeOperation` to filter without materializing all changes:
+
+```python
+with diff("old.jsonl", "new.jsonl", key="id") as result:
+    added = result.changes(ChangeOperation.ADDED)
+    for change in added:
+        print(change.key, change.new_line)
+```
+
+With numeric keys, API key components are `decimal.Decimal` values. For
+example, JSON identity `7` is returned as `Decimal("7")`; JSON strings and
+booleans retain their types. Details JSONL writes numeric keys as JSON numbers,
+including arbitrary-precision integers.
+
+### API errors
+
+The public error hierarchy starts with `JsonlDiffError`:
+
+- `ConfigurationError`: invalid keys, ignore pointers, or `max_temp`;
+- `InputError`: an invalid source or record; exposes `source` and optional
+  `line`;
+- `DuplicateKeyError`: an `InputError` with `key` and the first/repeated
+  physical lines in `lines`;
+- `ResourceError`: the temporary index cannot be created, written, or kept
+  within its configured limit.
+
+Failures during `diff()` clean up the workspace before the exception is
+raised.
+
+## Sources and compression
+
+Source opening and decompression are delegated to `py-jsonl`:
+
+| Source | CLI | Python API |
+|---|:---:|:---:|
+| Local path | Yes | Yes, including path-like values |
+| HTTP/HTTPS URL | Yes | Yes |
+| File-like object | No | Yes |
+| gzip (`.gz`) | Yes | Yes |
+| bzip2 (`.bz2`) | Yes | Yes |
+| xz (`.xz`) | Yes | Yes |
+| Zstandard (`.zst`) | Python 3.14 only | Python 3.14 only |
+| ZIP archive | No | No |
+
+Zstandard availability follows `py-jsonl` and its use of Python 3.14's
+standard-library zstd support; it is not supported by this project on earlier
+Python versions. Compression and HTTP transfer boundaries do not affect the
+reported decoded JSONL line numbers.
+
+## Comparison semantics
+
+### Strict JSONL records
+
+- Every physical line must be non-blank, valid JSON containing exactly one
+  top-level object.
+- Local paths and binary streams are decoded as UTF-8. Text streams are
+  already decoded, while URL response charset handling follows `py-jsonl`
+  (defaulting to UTF-8 when no charset is declared).
+- Blank lines, malformed JSON, duplicate object property names, `NaN`,
+  `Infinity`, and `-Infinity` are errors.
+- Top-level arrays, strings, numbers, booleans, and `null` are rejected.
+- Input errors identify `OLD` or `NEW` and include the physical line when
+  available.
+
+No normal summary or details output begins until both inputs pass complete
+input and duplicate validation.
+
+### Identity and duplicates
+
+Identity fields are top-level object-member names. Each component must exist
+and be a non-null string, number, or boolean; objects and arrays are invalid.
+Nested identity paths and automatic key detection are not supported.
+
+Composite identity field names are sorted lexically before their values are
+extracted. Consequently, `key=("b", "a")` and `key=("a", "b")` both produce
+keys in `(a, b)` order and match identically. Repeated or empty identity-field
+names are configuration errors.
+
+JSON types remain significant: `"1"` is different from `1`, and `true` is
+different from `1`.
+
+Every normalized identity must be unique within OLD and within NEW by default.
+The first duplicate aborts the comparison; `DuplicateKeyError` reports the
+first occurrence and repeated physical line. Set `duplicates="first"` to keep
+the first occurrence or `duplicates="last"` to keep the last occurrence and
+its physical line. These tolerant policies depend on physical input order.
+
+### Ignored object members
+
+Ignore expressions are exact
+[RFC 6901 JSON Pointers](https://www.rfc-editor.org/rfc/rfc6901). Escapes such
+as `~1` for `/` and `~0` for `~` are supported. An absent path is harmless,
+and `/updated_at` does not match `/metadata/updated_at`.
+
+Ignores are applied symmetrically before content canonicalization. A pointer
+may remove an entire array-valued object member, but it may not traverse an
+array or address an array element. Ignoring a top-level identity field is also
+invalid. Wildcards, JSONPath, and recursive name matching are not supported.
+
+### Canonical content and numbers
+
+Records with the same identity are compared after ignored members are removed:
+
+- JSON object property order is insignificant at every depth.
+- Array order is significant.
+- Unicode strings are compared by code-point sequence; no Unicode
+  normalization is applied.
+- Numbers are parsed as `Decimal` and compared by mathematical value.
+  `1`, `1.0`, and `1e0` are equal, and negative zero equals zero.
+- Arbitrary-precision integers and decimal values are not rounded through
+  binary floating point.
+
+Canonical content is stored with its length and SHA-256 digest. Matching
+canonical bytes are still compared, so a digest match alone is not treated as
+proof of equality. Numbers are written using compact scientific notation; a
+large exponent does not expand into a large string of zeroes.
+
+### Determinism
+
+Summary counts do not depend on either source's physical record order.
+Changes, including details records, use the stable lexical byte order of each
+canonical typed identity. This is a reproducibility guarantee, not numeric,
+locale-aware, or human-oriented sorting.
+
+## Disk-backed architecture and temporary files
+
+Each record is parsed and validated incrementally, normalized, and inserted
+into a private SQLite database under an operating-system temporary directory.
+The database stores the typed canonical identity, original line, canonical
+content, content length, and digest. A uniqueness constraint detects duplicate
+identities. SQL joins calculate the summary, and ordered SQLite cursors drive
+lazy change iteration.
+
+This architecture bounds memory by the records currently being processed and
+database buffers; it does not keep the complete decoded inputs or all changes
+in RAM. It does require temporary disk space. Canonical records and SQLite
+indexes remain until the `DiffResult` is closed, so temporary usage can exceed
+twice the combined decompressed input size.
+
+`max_temp` / `--max-temp` accepts a positive byte count. The implementation
+limits and checks files in the workspace owned by `jsonl-diff`, raising
+`ResourceError` (CLI exit `2`) when the index cannot stay within that budget.
+Choose a limit with room for SQLite pages and index overhead.
+
+`py-jsonl` may create its own temporary staging files for remote or compressed
+sources. Those files follow `py-jsonl`'s resource policy and are not counted by
+`jsonl-diff`'s `max_temp` limit. Total system temporary usage can therefore
+exceed the configured value.
+
+The private workspace is removed on context-manager exit, explicit `close()`,
+or a handled failure during construction. Cleanup failures are emitted as
+warnings rather than replacing the primary error.
+
+## Limitations and non-goals
+
+`jsonl-diff` intentionally does not provide:
+
+- line- or physical-order comparison;
+- equal-record events or complete record payloads in details output;
+- field-level diffs, JSON Patch, move detection, or rename detection;
+- nested identities, fuzzy matching, or key autodetection;
+- numeric tolerances, unordered-array comparison, or Unicode normalization;
+- wildcard/JSONPath ignores or ignores that traverse arrays;
+- schema validation or repair of malformed input;
+- ZIP input, database-table input, cloud-provider SDK integrations, GUI, or
+  HTML reports.
+
+It is a reconciliation tool for strict JSONL datasets, not a general-purpose
+visual JSON diff.
+
+## Development
+
+Clone the repository and sync the test and lint dependency groups with `uv`:
+
+```bash
+uv sync --group test --group lint
+```
+
+Run the test suite and Ruff:
+
+```bash
+uv run pytest
+uv run ruff check --quiet --output-format=concise .
+```
+
+The tests cover CLI behavior, identity and canonicalization rules, strict
+validation, details output, source types, compression, temporary limits, and
+resource cleanup.
+
+## License
+
+See [LICENSE](LICENSE).

jsonl_diff-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+jsonl_diff.py,sha256=iupZ9h0SIG8gBXy3zjabQznv6HsVaE1q2FDKs00Zb0s,24649
+jsonl_diff-0.1.0.dist-info/licenses/LICENSE,sha256=awYAatdjruzs9zNPe2GuP-Ozwb4gCm6FKt0BIY4DkmQ,1078
+jsonl_diff-0.1.0.dist-info/METADATA,sha256=dwyZrMBhSJQW8xvkZ46vUrOT6taXKKXmcwobsK7rx7U,19991
+jsonl_diff-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+jsonl_diff-0.1.0.dist-info/entry_points.txt,sha256=cRg5gUGYOAC64jrkvuo9GCEx66fwmYTtogwa73PT9Bw,47
+jsonl_diff-0.1.0.dist-info/top_level.txt,sha256=xdbZTzW96-R8MmntMHqF1hSYJjsxSHU3xNdvVPlJiKo,11
+jsonl_diff-0.1.0.dist-info/RECORD,,

jsonl_diff-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

jsonl_diff-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+jsonl-diff = jsonl_diff:main

jsonl_diff-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Rolando Morales Perez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

jsonl_diff-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+jsonl_diff

jsonl_diff.py

@@ -0,0 +1,723 @@
+"""Strict, disk-backed reconciliation of JSON Lines datasets."""
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import sqlite3
+import sys
+import tempfile
+import warnings
+from dataclasses import dataclass
+from decimal import Decimal
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, Iterable, Iterator, Optional, Sequence, Tuple, Union
+
+import jsonl
+
+Scalar = Union[str, Decimal, int, float, bool]
+IdentityKey = Tuple[Union[str, Decimal, bool], ...]
+
+_MISSING = object()
+
+
+class JsonlDiffError(Exception):
+    """Base exception for jsonl-diff failures."""
+
+
+class ConfigurationError(JsonlDiffError, ValueError):
+    """Raised when comparison options are invalid."""
+
+
+class InputError(JsonlDiffError):
+    """Raised when an input source or record is invalid."""
+
+    def __init__(self, message: str, source: str, line: Optional[int] = None):
+        self.source = source
+        self.line = line
+        location = " in {}".format(source)
+        if line is not None:
+            location += " at line {}".format(line)
+        super().__init__("{}{}".format(message, location))
+
+
+class DuplicateKeyError(InputError):
+    """Raised when an identity occurs more than once in one source."""
+
+    def __init__(self, key: IdentityKey, source: str, lines: Sequence[int]):
+        self.key = key
+        self.lines = tuple(lines)
+
+        message = "duplicate key {} on lines {}".format(
+            _canonical_text(list(key), ensure_ascii=True),
+            ", ".join(str(line) for line in lines),
+        )
+        super().__init__(message, source)
+
+
+class ResourceError(JsonlDiffError):
+    """Raised when a resource limit prevents comparison."""
+
+
+class ChangeOperation(str, Enum):
+    """A changed record classification."""
+
+    ADDED = "added"
+    DELETED = "deleted"
+    MODIFIED = "modified"
+
+
+class DuplicatePolicy(str, Enum):
+    """How duplicate identities are handled."""
+
+    ERROR = "error"
+    FIRST = "first"
+    LAST = "last"
+
+
+@dataclass(frozen=True)
+class SourceLocation:
+    """A physical source location."""
+
+    source: str
+    line: int
+
+
+@dataclass(frozen=True)
+class Summary:
+    """Comparison totals."""
+
+    equal: int
+    added: int
+    deleted: int
+    modified: int
+
+    @property
+    def different(self) -> bool:
+        """Return whether any changed records were found."""
+        return bool(self.added or self.deleted or self.modified)
+
+
+@dataclass(frozen=True)
+class DiffConfig:
+    """Normalized comparison configuration."""
+
+    key: Tuple[str, ...]
+    ignore: Tuple[str, ...] = ()
+    duplicates: DuplicatePolicy = DuplicatePolicy.ERROR
+    max_temp: Optional[int] = None
+
+
+@dataclass(frozen=True)
+class Change:
+    """One added, deleted, or modified identity."""
+
+    operation: ChangeOperation
+    key: IdentityKey
+    old: Optional[SourceLocation]
+    new: Optional[SourceLocation]
+
+    @property
+    def old_line(self) -> Optional[int]:
+        """Return the OLD physical line, when present."""
+        return None if self.old is None else self.old.line
+
+    @property
+    def new_line(self) -> Optional[int]:
+        """Return the NEW physical line, when present."""
+        return None if self.new is None else self.new.line
+
+
+def _reject_constant(value: str) -> None:
+    raise ValueError("invalid JSON number {!r}".format(value))
+
+
+def _object(pairs: Iterable[Tuple[str, Any]]) -> Dict[str, Any]:
+    result = {}
+    for name, value in pairs:
+        if name in result:
+            raise ValueError("duplicate property {!r}".format(name))
+        result[name] = value
+    return result
+
+
+def _decode_json(value: str) -> Any:
+    return json.loads(
+        value,
+        parse_int=Decimal,
+        parse_float=Decimal,
+        parse_constant=_reject_constant,
+        object_pairs_hook=_object,
+    )
+
+
+def _number(value: Scalar) -> str:
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise ValueError("non-finite numbers are not valid JSON")
+        value = Decimal(str(value))
+    elif isinstance(value, int):
+        value = Decimal(value)
+    if not isinstance(value, Decimal) or not value.is_finite():
+        raise ValueError("non-finite numbers are not valid JSON")
+    if value.is_zero():
+        return "0"
+
+    sign, digits, exponent = value.as_tuple()
+    trailing_zeros = 0
+    while len(digits) - trailing_zeros > 1 and digits[-trailing_zeros - 1] == 0:
+        trailing_zeros += 1
+
+    digits = digits[:-trailing_zeros] if trailing_zeros else digits
+    exponent += trailing_zeros
+    mantissa = str(digits[0])
+    if len(digits) > 1:
+        mantissa += "." + "".join(str(digit) for digit in digits[1:])
+
+    scientific_exponent = exponent + len(digits) - 1
+    return "{}{}e{:+d}".format("-" if sign else "", mantissa, scientific_exponent)
+
+
+def _canonical_text(value: Any, ensure_ascii: bool = False) -> str:
+    if value is None:
+        return "null"
+    if value is True:
+        return "true"
+    if value is False:
+        return "false"
+    if isinstance(value, (Decimal, int, float)):
+        return _number(value)
+    if isinstance(value, str):
+        return json.dumps(value, ensure_ascii=ensure_ascii)
+    if isinstance(value, (list, tuple)):
+        return "[{}]".format(",".join(_canonical_text(item, ensure_ascii) for item in value))
+    if isinstance(value, dict):
+        if any(not isinstance(name, str) for name in value):
+            raise ValueError("JSON object property names must be strings")
+        members = (
+            "{}:{}".format(
+                json.dumps(name, ensure_ascii=ensure_ascii),
+                _canonical_text(value[name], ensure_ascii),
+            )
+            for name in sorted(value)
+        )
+        return "{{{}}}".format(",".join(members))
+    raise ValueError("unsupported JSON value type: {}".format(type(value).__name__))
+
+
+def _canonical(value: Any) -> bytes:
+    return _canonical_text(value).encode("utf-8")
+
+
+def _fingerprint(canonical: bytes) -> Tuple[int, bytes]:
+    return len(canonical), hashlib.sha256(canonical).digest()
+
+
+def _parse_pointer(pointer: str) -> Tuple[str, ...]:
+    if not pointer.startswith("/") or pointer == "/":
+        if pointer == "/":
+            return ("",)
+        raise ConfigurationError("ignore paths must be non-empty RFC 6901 JSON Pointers")
+    tokens = []
+    for token in pointer[1:].split("/"):
+        index = 0
+        decoded = []
+        while index < len(token):
+            if token[index] != "~":
+                decoded.append(token[index])
+                index += 1
+            elif index + 1 >= len(token) or token[index + 1] not in "01":
+                raise ConfigurationError("invalid JSON Pointer escape in {!r}".format(pointer))
+            else:
+                decoded.append("/" if token[index + 1] == "1" else "~")
+                index += 2
+        tokens.append("".join(decoded))
+    return tuple(tokens)
+
+
+def _ignore_tree(paths: Sequence[str], keys: Sequence[str]) -> Dict[str, Any]:
+    tree = {}
+    for pointer in paths:
+        tokens = _parse_pointer(pointer)
+        if len(tokens) == 1 and tokens[0] in keys:
+            raise ConfigurationError("identity field {!r} cannot be ignored".format(tokens[0]))
+        node = tree
+        for token in tokens:
+            node = node.setdefault(token, {})
+        node[_MISSING] = True
+    return tree
+
+
+def _remove_ignored(value: Any, tree: Dict[str, Any]) -> Any:
+    if not tree:
+        return value
+    if isinstance(value, list):
+        raise ValueError("ignore paths may not traverse arrays")
+    if not isinstance(value, dict):
+        return value
+    result = {}
+    for name, item in value.items():
+        child = tree.get(name)
+        if child is None:
+            result[name] = item
+        elif _MISSING not in child:
+            result[name] = _remove_ignored(item, child)
+    return result
+
+
+def _identity(record: Dict[str, Any], keys: Sequence[str]) -> IdentityKey:
+    values = []
+    for name in keys:
+        if name not in record:
+            raise ValueError("missing identity field {!r}".format(name))
+        value = record[name]
+        if value is None or not isinstance(value, (str, Decimal, int, float, bool)):
+            raise ValueError("identity field {!r} must be a non-null scalar".format(name))
+        values.append(value)
+    return tuple(values)
+
+
+def _records(source: Any, on_error: Any) -> Iterator[Any]:
+    yield from jsonl.load(
+        source,
+        parse_int=Decimal,
+        parse_float=Decimal,
+        parse_constant=_reject_constant,
+        object_pairs_hook=_object,
+        _on_error=on_error,
+    )
+
+
+class DiffResult:
+    """A context-managed, disk-backed comparison result."""
+
+    def __init__(self, old: Any, new: Any, config: DiffConfig):
+        self._old = old
+        self._new = new
+        self.config = config
+        self._ignore_tree = _ignore_tree(config.ignore, config.key)
+        self._workspace = None
+        self._connection = None
+        self._summary_value = None
+        self._started = False
+
+    def _open(self) -> None:
+        try:
+            self._workspace = tempfile.TemporaryDirectory(prefix="jsonl-diff-")
+            self._connection = sqlite3.connect(os.path.join(self._workspace.name, "index.sqlite3"))
+            self._configure_database()
+            self._create_schema()
+        except (OSError, sqlite3.Error) as error:
+            self.close()
+            raise ResourceError("could not create the temporary index") from error
+
+    def _configure_database(self) -> None:
+        self._connection.execute("PRAGMA journal_mode = OFF")
+        self._connection.execute("PRAGMA temp_store = FILE")
+        if self.config.max_temp is not None:
+            pages = max(1, self.config.max_temp // 4096)
+            self._connection.execute("PRAGMA page_size = 4096")
+            self._connection.execute("PRAGMA max_page_count = {}".format(pages))
+
+    def _create_schema(self) -> None:
+        self._connection.execute(
+            """
+            CREATE TABLE records (
+                side INTEGER NOT NULL,
+                identity BLOB NOT NULL,
+                line INTEGER NOT NULL,
+                length INTEGER NOT NULL,
+                digest BLOB NOT NULL,
+                canonical BLOB NOT NULL,
+                PRIMARY KEY (side, identity)
+            ) WITHOUT ROWID
+            """,
+        )
+
+    def _require_summary(self) -> Summary:
+        if self._summary_value is None:
+            raise RuntimeError("diff() must be used as a context manager")
+        return self._summary_value
+
+    @property
+    def summary(self) -> Summary:
+        """Return comparison totals after the context has been entered."""
+        return self._require_summary()
+
+    @property
+    def equal(self) -> int:
+        return self._require_summary().equal
+
+    @property
+    def added(self) -> int:
+        return self._require_summary().added
+
+    @property
+    def deleted(self) -> int:
+        return self._require_summary().deleted
+
+    @property
+    def modified(self) -> int:
+        return self._require_summary().modified
+
+    @property
+    def different(self) -> bool:
+        return self._require_summary().different
+
+    def changes(self, operation: Optional[ChangeOperation] = None) -> Iterator[Change]:
+        """Iterate changed identities in deterministic order."""
+        if self._connection is None:
+            raise RuntimeError("the diff result is closed")
+        requested = None if operation is None else ChangeOperation(operation)
+        query = """
+            SELECT identity, line, length, digest, canonical
+            FROM records
+            WHERE side = ?
+            ORDER BY identity
+        """
+        old_records = iter(self._connection.execute(query, (0,)))
+        new_records = iter(self._connection.execute(query, (1,)))
+        old_record = next(old_records, None)
+        new_record = next(new_records, None)
+        while old_record is not None or new_record is not None:
+            comparison = (
+                1
+                if old_record is None
+                else -1
+                if new_record is None
+                else (old_record[0] > new_record[0]) - (old_record[0] < new_record[0])
+            )
+            if comparison < 0:
+                change = self._change(ChangeOperation.DELETED, old_record, None)
+                old_record = next(old_records, None)
+            elif comparison > 0:
+                change = self._change(ChangeOperation.ADDED, None, new_record)
+                new_record = next(new_records, None)
+            else:
+                old_signature = old_record[2:]
+                new_signature = new_record[2:]
+                change = (
+                    None
+                    if old_signature == new_signature
+                    else self._change(ChangeOperation.MODIFIED, old_record, new_record)
+                )
+                old_record = next(old_records, None)
+                new_record = next(new_records, None)
+            if change is not None and (requested is None or change.operation == requested):
+                yield change
+
+    def _change(
+        self,
+        operation: ChangeOperation,
+        old_record: Optional[Tuple[Any, ...]],
+        new_record: Optional[Tuple[Any, ...]],
+    ) -> Change:
+        record = old_record or new_record
+        key = tuple(_decode_json(record[0].decode("utf-8")))
+        old_line = None if old_record is None else old_record[1]
+        new_line = None if new_record is None else new_record[1]
+        return Change(
+            operation,
+            key,
+            None if old_line is None else SourceLocation("OLD", old_line),
+            None if new_line is None else SourceLocation("NEW", new_line),
+        )
+
+    def close(self) -> None:
+        """Release database and temporary files."""
+        if self._connection is not None:
+            try:
+                self._connection.close()
+            except sqlite3.Error as error:
+                warnings.warn(
+                    "could not close temporary database: {}".format(error),
+                    stacklevel=2,
+                )
+            self._connection = None
+        if self._workspace is not None:
+            try:
+                self._workspace.cleanup()
+            except OSError as error:
+                warnings.warn(
+                    "could not remove temporary files: {}".format(error),
+                    stacklevel=2,
+                )
+            self._workspace = None
+
+    def __enter__(self) -> "DiffResult":
+        if self._started:
+            raise RuntimeError("the diff result cannot be entered more than once")
+        self._started = True
+        try:
+            self._open()
+            self._index(self._old, 0)
+            self._index(self._new, 1)
+            self._summary_value = self._calculate_summary()
+            return self
+        except BaseException:
+            self.close()
+            raise
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.close()
+
+    def _insert_records(self, records: Iterable[Any], side: int, source: str) -> None:
+        cursor = self._connection.cursor()
+        insert = {
+            DuplicatePolicy.ERROR: "INSERT",
+            DuplicatePolicy.FIRST: "INSERT OR IGNORE",
+            DuplicatePolicy.LAST: "INSERT OR REPLACE",
+        }[self.config.duplicates]
+        for line, record in enumerate(records, start=1):
+            if not isinstance(record, dict):
+                raise InputError("each record must be a JSON object", source, line)
+            try:
+                key = _identity(record, self.config.key)
+                normalized = _remove_ignored(record, self._ignore_tree)
+                canonical = _canonical(normalized)
+                identity = _canonical(list(key))
+                length, digest = _fingerprint(canonical)
+            except (TypeError, ValueError) as error:
+                raise InputError(str(error), source, line) from error
+            try:
+                cursor.execute(
+                    "{} INTO records VALUES (?, ?, ?, ?, ?, ?)".format(insert),
+                    (
+                        side,
+                        identity,
+                        line,
+                        length,
+                        digest,
+                        canonical,
+                    ),
+                )
+            except sqlite3.IntegrityError as error:
+                first = cursor.execute(
+                    "SELECT line FROM records WHERE side = ? AND identity = ?",
+                    (side, identity),
+                ).fetchone()[0]
+                raise DuplicateKeyError(key, source, (first, line)) from error
+            except sqlite3.DatabaseError as error:
+                raise ResourceError("could not write the temporary index") from error
+            if self.config.max_temp is not None:
+                self._check_size()
+        self._check_size()
+
+    def _index(self, source: Any, side: int) -> None:
+        name = "OLD" if side == 0 else "NEW"
+        error_line = [None]
+
+        def on_error(line: int, error: Exception) -> None:
+            error_line[0] = line
+
+        try:
+            with self._connection:
+                self._insert_records(_records(source, on_error), side, name)
+        except JsonlDiffError:
+            raise
+        except (OSError, EOFError, ValueError, RuntimeError) as error:
+            line = error_line[0]
+            message = "invalid input ({})".format(type(error).__name__)
+            raise InputError(message, name, line) from error
+        except sqlite3.DatabaseError as error:
+            raise ResourceError("could not build the temporary index") from error
+
+    def _check_size(self) -> None:
+        if self.config.max_temp is None:
+            return
+        usage = sum(
+            entry.stat().st_size
+            for entry in Path(self._workspace.name).iterdir()
+            if entry.is_file()
+        )
+        if usage > self.config.max_temp:
+            raise ResourceError(
+                "temporary storage exceeded {}".format(self.config.max_temp),
+            )
+
+    def _calculate_summary(self) -> Summary:
+        row = self._connection.execute(
+            """
+            SELECT
+                (SELECT COUNT(*)
+                 FROM records AS o JOIN records AS n USING (identity)
+                 WHERE o.side = 0 AND n.side = 1
+                   AND o.length = n.length
+                   AND o.digest = n.digest
+                   AND o.canonical = n.canonical),
+                (SELECT COUNT(*)
+                 FROM records AS n
+                 WHERE n.side = 1 AND NOT EXISTS (
+                     SELECT 1 FROM records AS o
+                     WHERE o.side = 0 AND o.identity = n.identity
+                 )),
+                (SELECT COUNT(*)
+                 FROM records AS o
+                 WHERE o.side = 0 AND NOT EXISTS (
+                     SELECT 1 FROM records AS n
+                     WHERE n.side = 1 AND n.identity = o.identity
+                 )),
+                (SELECT COUNT(*)
+                 FROM records AS o JOIN records AS n USING (identity)
+                 WHERE o.side = 0 AND n.side = 1
+                   AND (
+                       o.length != n.length
+                       OR o.digest != n.digest
+                       OR o.canonical != n.canonical
+                   ))
+            """,
+        ).fetchone()
+        return Summary(*(int(value or 0) for value in row))
+
+
+def _configuration(
+    key: Union[str, Sequence[str]],
+    ignore: Sequence[str],
+    duplicates: Union[str, DuplicatePolicy],
+    max_temp: Optional[int],
+) -> DiffConfig:
+    keys = (key,) if isinstance(key, str) else tuple(key)
+    if not keys or any(not isinstance(name, str) or not name for name in keys):
+        raise ConfigurationError("at least one non-empty identity field is required")
+    if len(set(keys)) != len(keys):
+        raise ConfigurationError("identity fields must not be repeated")
+    keys = tuple(sorted(keys))
+    ignores = tuple(dict.fromkeys(ignore))
+    _ignore_tree(ignores, keys)
+    try:
+        duplicate_policy = DuplicatePolicy(duplicates)
+    except ValueError as error:
+        raise ConfigurationError("invalid duplicate policy {!r}".format(duplicates)) from error
+    if (
+        max_temp is not None
+        and (not isinstance(max_temp, int) or isinstance(max_temp, bool) or max_temp <= 0)
+    ):
+        raise ConfigurationError("max_temp must be a positive integer")
+    return DiffConfig(keys, ignores, duplicate_policy, max_temp)
+
+
+def diff(
+    old: Any,
+    new: Any,
+    *,
+    key: Union[str, Sequence[str]],
+    ignore: Sequence[str] = (),
+    duplicates: Union[str, DuplicatePolicy] = DuplicatePolicy.ERROR,
+    max_temp: Optional[int] = None,
+) -> DiffResult:
+    """Create a context-managed, disk-backed comparison."""
+    config = _configuration(key, ignore, duplicates, max_temp)
+    return DiffResult(old, new, config)
+
+
+def _change_dict(change: Change) -> Dict[str, Any]:
+    result = {
+        "type": "change",
+        "op": change.operation.value,
+        "key": list(change.key),
+    }
+    if change.old_line is not None:
+        result["old_line"] = change.old_line
+    if change.new_line is not None:
+        result["new_line"] = change.new_line
+    return result
+
+
+def _write_text(result: DiffResult) -> None:
+    summary = result.summary
+    print("Records:")
+    print("  equal:     {:,}".format(summary.equal))
+    print("  added:     {:,}".format(summary.added))
+    print("  deleted:   {:,}".format(summary.deleted))
+    print("  modified:  {:,}".format(summary.modified))
+
+
+def _write_details(result: DiffResult, path: Union[str, os.PathLike]) -> None:
+    def events() -> Iterator[Dict[str, Any]]:
+        yield {
+            "type": "meta",
+            "key": list(result.config.key),
+            "ignore": list(result.config.ignore),
+            "duplicates": result.config.duplicates.value,
+        }
+        for change in result.changes():
+            yield _change_dict(change)
+        yield {
+            "type": "summary",
+            "equal": result.equal,
+            "added": result.added,
+            "deleted": result.deleted,
+            "modified": result.modified,
+        }
+
+    jsonl.dump(events(), path, cls=_canonical_text)
+
+
+class _ArgumentParser(argparse.ArgumentParser):
+    def error(self, message: str) -> None:
+        self.print_usage(sys.stderr)
+        self.exit(3, "{}: error: {}\n".format(self.prog, message))
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = _ArgumentParser(prog="jsonl-diff")
+    parser.add_argument("old")
+    parser.add_argument("new")
+    parser.add_argument("--key", action="append", required=True)
+    parser.add_argument("--ignore", action="append", default=[])
+    parser.add_argument(
+        "--duplicates",
+        choices=tuple(policy.value for policy in DuplicatePolicy),
+        default=DuplicatePolicy.ERROR.value,
+    )
+    parser.add_argument("--details", metavar="FILE")
+    parser.add_argument("--quiet", action="store_true")
+    parser.add_argument("--max-temp", type=int)
+    return parser
+
+
+def _run_comparison(arguments: argparse.Namespace, old: Any, new: Any, keys: Tuple[str, ...]) -> int:
+    with diff(
+        old,
+        new,
+        key=keys,
+        ignore=arguments.ignore,
+        duplicates=arguments.duplicates,
+        max_temp=arguments.max_temp,
+    ) as result:
+        if arguments.details:
+            _write_details(result, arguments.details)
+        if not arguments.quiet:
+            _write_text(result)
+            sys.stdout.flush()
+        return 1 if result.different else 0
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    """Run the jsonl-diff command-line interface."""
+    parser = _parser()
+    arguments = parser.parse_args(argv)
+    if arguments.old == "-" and arguments.new == "-":
+        parser.error("OLD and NEW cannot both read from stdin")
+    old = sys.stdin.buffer if arguments.old == "-" else arguments.old
+    new = sys.stdin.buffer if arguments.new == "-" else arguments.new
+    keys = tuple(name for item in arguments.key for name in item.split(","))
+    try:
+        return _run_comparison(arguments, old, new, keys)
+    except ConfigurationError as error:
+        parser.error(str(error))
+    except (InputError, ResourceError) as error:
+        print("ERROR: {}".format(error), file=sys.stderr)
+        return 2
+    except OSError as error:
+        print("ERROR: output failed ({})".format(type(error).__name__), file=sys.stderr)
+        return 2
+    except KeyboardInterrupt:
+        print("ERROR: interrupted", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())