jaro 0.1.15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jaro-0.1.15/.gitignore +7 -0
- jaro-0.1.15/PKG-INFO +74 -0
- jaro-0.1.15/README.md +57 -0
- jaro-0.1.15/bench/bench.py +147 -0
- jaro-0.1.15/bench/decode_baseline.json +14 -0
- jaro-0.1.15/bench/decode_gm.py +60 -0
- jaro-0.1.15/bench/decode_metric.py +68 -0
- jaro-0.1.15/bench/decode_ratio.py +107 -0
- jaro-0.1.15/bench/ratio_baseline.json +14 -0
- jaro-0.1.15/bench/results.json +134 -0
- jaro-0.1.15/experiments-decode.jsonl +29 -0
- jaro-0.1.15/experiments-decode2.jsonl +31 -0
- jaro-0.1.15/experiments-ratio.jsonl +10 -0
- jaro-0.1.15/pyproject.toml +36 -0
- jaro-0.1.15/src/jaro/__init__.py +10 -0
- jaro-0.1.15/src/jaro/_core.py +115 -0
- jaro-0.1.15/src/jaro/binary.py +714 -0
- jaro-0.1.15/src/jaro/columns.py +159 -0
- jaro-0.1.15/src/jaro/dedup.py +80 -0
- jaro-0.1.15/src/jaro/errors.py +17 -0
- jaro-0.1.15/src/jaro/py.typed +0 -0
- jaro-0.1.15/src/jaro/tables.py +155 -0
- jaro-0.1.15/src/jaro/text.py +927 -0
- jaro-0.1.15/tests/conftest.py +27 -0
- jaro-0.1.15/tests/test_conformance.py +112 -0
- jaro-0.1.15/tests/test_corpus.py +25 -0
- jaro-0.1.15/tests/test_decode_binary_strings.py +120 -0
- jaro-0.1.15/tests/test_decode_differential.py +204 -0
- jaro-0.1.15/tests/test_decode_fastpaths.py +141 -0
- jaro-0.1.15/tests/test_decode_strings.py +72 -0
- jaro-0.1.15/tests/test_decode_tables.py +150 -0
- jaro-0.1.15/tests/test_edges.py +480 -0
- jaro-0.1.15/tests/test_roundtrip.py +162 -0
jaro-0.1.15/.gitignore
ADDED
jaro-0.1.15/PKG-INFO
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: jaro
|
|
3
|
+
Version: 0.1.15
|
|
4
|
+
Summary: JARO text and binary data format (pure Python)
|
|
5
|
+
Project-URL: Homepage, https://jaro.raczylo.com
|
|
6
|
+
Project-URL: Repository, https://github.com/lukaszraczylo/jaro
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: mypy==2.4.0; extra == 'dev'
|
|
11
|
+
Requires-Dist: pytest==9.1.1; extra == 'dev'
|
|
12
|
+
Requires-Dist: pyyaml==6.0.3; extra == 'dev'
|
|
13
|
+
Requires-Dist: types-pyyaml==6.0.12.20260906; extra == 'dev'
|
|
14
|
+
Provides-Extra: native
|
|
15
|
+
Requires-Dist: jaro-native; extra == 'native'
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# jaro (Python)
|
|
19
|
+
|
|
20
|
+
Pure-Python codec for JARO, a JSON-model data format with a text form and a binary form. The format is specified in [`../SPEC.md`](../SPEC.md). Python 3.10 or later, no runtime dependencies, fully typed.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
pip install . # from this directory
|
|
26
|
+
pip install -e '.[dev]' # tests, mypy and the YAML benchmark
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## API
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
import jaro
|
|
33
|
+
|
|
34
|
+
jaro.dumps({"a": [1, 2.5, None]}) # 'a:[1 2.5 ~]'
|
|
35
|
+
jaro.loads("a:[1 2.5 ~]") # {'a': [1, 2.5, None]}
|
|
36
|
+
jaro.dumps_binary(value, dedup=False) # bytes
|
|
37
|
+
jaro.loads_binary(data) # value
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
- `dumps(value, *, dedup=False) -> str` and `loads(text: str | bytes) -> Any`.
|
|
41
|
+
- `dumps_binary(value, *, dedup=False) -> bytes` and `loads_binary(data: bytes) -> Any`.
|
|
42
|
+
- `dedup=True` replaces repeated sub-trees (24 bytes or more of text) with anchors and references.
|
|
43
|
+
- `JaroError` subclasses `ValueError`. Its `offset` attribute is the byte offset in the input, or `None` for encoder errors.
|
|
44
|
+
|
|
45
|
+
Mapping: `None`, `bool`, `int` (int64), `float` (finite), `str`, `list` and `dict` with `str` keys. Encoders sort keys. Decoders reject duplicate keys. Integral floats below 2^53 decode as `int`, and zero is always `0`. Tuples, sets and bytes are rejected.
|
|
46
|
+
|
|
47
|
+
Supported: text and binary forms, anchors, spreads, comments, tables (plain, sparse with `^`, keyed `{|k| ...}`, nested columns `key{sub}`). Not supported: schema validation, the `Parse`/`Doc` arena, HTTP middleware and zstd.
|
|
48
|
+
|
|
49
|
+
## Conformance
|
|
50
|
+
|
|
51
|
+
`tests/test_conformance.py` runs `../conformance/vectors.json`: every `text`, `binary`, `badText`, `badTextHex` and `badBinaryHex` case, a round trip of every `trees` entry, and byte-exact parity with the Go reference for every vector that has a tree (text, binary and binary with dedup). The parity allowlists in that file are empty.
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
pytest -q
|
|
55
|
+
mypy --strict src
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Limits
|
|
59
|
+
|
|
60
|
+
- Nesting depth 512, reference expansion 2^24 nodes, binary counts checked against the remaining input and a node budget of 4M plus 32 per input byte. Codecs use about one Python frame per nesting level, so depth 512 works under the default recursion limit. A `RecursionError` from a very deep caller stack is reported as `JaroError`.
|
|
61
|
+
- Invalid UTF-8 is rejected on decode. A lone surrogate in a `str` is rejected on encode.
|
|
62
|
+
- Binary floats that are NaN or infinite are rejected on decode.
|
|
63
|
+
|
|
64
|
+
## Benchmark
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
python bench/bench.py # corpus: $JARO_CORPUS, default ../../corpus
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
It prints encode and decode milliseconds and output bytes against `json` and PyYAML (libyaml loader when available), and writes `bench/results.json`.
|
|
71
|
+
|
|
72
|
+
## Optional native decoder
|
|
73
|
+
|
|
74
|
+
`loads_binary` uses the `jaro_native` Rust extension when it is installed (`pip install jaro-native`, built from `native-py/`). Without it, the pure Python decoder runs. Set `JARO_PURE=1` to force the pure decoder. `jaro.binary.NATIVE` reports which one is active. `jaro.binary.loads_binary_py` always runs the pure decoder.
|
jaro-0.1.15/README.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# jaro (Python)
|
|
2
|
+
|
|
3
|
+
Pure-Python codec for JARO, a JSON-model data format with a text form and a binary form. The format is specified in [`../SPEC.md`](../SPEC.md). Python 3.10 or later, no runtime dependencies, fully typed.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
pip install . # from this directory
|
|
9
|
+
pip install -e '.[dev]' # tests, mypy and the YAML benchmark
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## API
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
import jaro
|
|
16
|
+
|
|
17
|
+
jaro.dumps({"a": [1, 2.5, None]}) # 'a:[1 2.5 ~]'
|
|
18
|
+
jaro.loads("a:[1 2.5 ~]") # {'a': [1, 2.5, None]}
|
|
19
|
+
jaro.dumps_binary(value, dedup=False) # bytes
|
|
20
|
+
jaro.loads_binary(data) # value
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
- `dumps(value, *, dedup=False) -> str` and `loads(text: str | bytes) -> Any`.
|
|
24
|
+
- `dumps_binary(value, *, dedup=False) -> bytes` and `loads_binary(data: bytes) -> Any`.
|
|
25
|
+
- `dedup=True` replaces repeated sub-trees (24 bytes or more of text) with anchors and references.
|
|
26
|
+
- `JaroError` subclasses `ValueError`. Its `offset` attribute is the byte offset in the input, or `None` for encoder errors.
|
|
27
|
+
|
|
28
|
+
Mapping: `None`, `bool`, `int` (int64), `float` (finite), `str`, `list` and `dict` with `str` keys. Encoders sort keys. Decoders reject duplicate keys. Integral floats below 2^53 decode as `int`, and zero is always `0`. Tuples, sets and bytes are rejected.
|
|
29
|
+
|
|
30
|
+
Supported: text and binary forms, anchors, spreads, comments, tables (plain, sparse with `^`, keyed `{|k| ...}`, nested columns `key{sub}`). Not supported: schema validation, the `Parse`/`Doc` arena, HTTP middleware and zstd.
|
|
31
|
+
|
|
32
|
+
## Conformance
|
|
33
|
+
|
|
34
|
+
`tests/test_conformance.py` runs `../conformance/vectors.json`: every `text`, `binary`, `badText`, `badTextHex` and `badBinaryHex` case, a round trip of every `trees` entry, and byte-exact parity with the Go reference for every vector that has a tree (text, binary and binary with dedup). The parity allowlists in that file are empty.
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
pytest -q
|
|
38
|
+
mypy --strict src
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Limits
|
|
42
|
+
|
|
43
|
+
- Nesting depth 512, reference expansion 2^24 nodes, binary counts checked against the remaining input and a node budget of 4M plus 32 per input byte. Codecs use about one Python frame per nesting level, so depth 512 works under the default recursion limit. A `RecursionError` from a very deep caller stack is reported as `JaroError`.
|
|
44
|
+
- Invalid UTF-8 is rejected on decode. A lone surrogate in a `str` is rejected on encode.
|
|
45
|
+
- Binary floats that are NaN or infinite are rejected on decode.
|
|
46
|
+
|
|
47
|
+
## Benchmark
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
python bench/bench.py # corpus: $JARO_CORPUS, default ../../corpus
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
It prints encode and decode milliseconds and output bytes against `json` and PyYAML (libyaml loader when available), and writes `bench/results.json`.
|
|
54
|
+
|
|
55
|
+
## Optional native decoder
|
|
56
|
+
|
|
57
|
+
`loads_binary` uses the `jaro_native` Rust extension when it is installed (`pip install jaro-native`, built from `native-py/`). Without it, the pure Python decoder runs. Set `JARO_PURE=1` to force the pure decoder. `jaro.binary.NATIVE` reports which one is active. `jaro.binary.loads_binary_py` always runs the pure decoder.
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Time JARO against json and PyYAML on a directory of real JSON files.
|
|
2
|
+
|
|
3
|
+
Usage: python bench/bench.py [--budget SECONDS] [--yaml auto|c|pure]
|
|
4
|
+
The corpus is $JARO_CORPUS, default ../../corpus relative to py/.
|
|
5
|
+
Writes bench/results.json as {file: {format: {enc_ms, dec_ms, bytes}}}.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
PY_ROOT = Path(__file__).resolve().parent.parent
|
|
20
|
+
sys.path.insert(0, str(PY_ROOT / "src"))
|
|
21
|
+
sys.path.insert(0, str(PY_ROOT / "tests"))
|
|
22
|
+
|
|
23
|
+
from conftest import same # noqa: E402
|
|
24
|
+
|
|
25
|
+
import jaro # noqa: E402
|
|
26
|
+
|
|
27
|
+
MIN_RUNS = 3
|
|
28
|
+
MAX_RUNS = 200
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def best_ms(fn: Callable[[], Any], budget: float) -> float:
|
|
32
|
+
"""Return the fastest run in ms: at least MIN_RUNS runs, then stop after budget seconds."""
|
|
33
|
+
best = float("inf")
|
|
34
|
+
start = time.perf_counter()
|
|
35
|
+
runs = 0
|
|
36
|
+
while runs < MIN_RUNS or (runs < MAX_RUNS and time.perf_counter() - start < budget):
|
|
37
|
+
t0 = time.perf_counter()
|
|
38
|
+
fn()
|
|
39
|
+
best = min(best, time.perf_counter() - t0)
|
|
40
|
+
runs += 1
|
|
41
|
+
return best * 1000
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def yaml_codec(
|
|
45
|
+
choice: str,
|
|
46
|
+
) -> tuple[str, Callable[[bytes], Any], Callable[[Any], str]] | None:
|
|
47
|
+
try:
|
|
48
|
+
import yaml # type: ignore[import-untyped,unused-ignore]
|
|
49
|
+
except ImportError:
|
|
50
|
+
return None
|
|
51
|
+
c_ok = bool(getattr(yaml, "__with_libyaml__", False))
|
|
52
|
+
if choice == "c" and not c_ok:
|
|
53
|
+
raise SystemExit("libyaml is not available in this PyYAML build")
|
|
54
|
+
if (choice in ("auto", "c")) and c_ok:
|
|
55
|
+
return (
|
|
56
|
+
"PyYAML CSafeLoader/CSafeDumper (libyaml)",
|
|
57
|
+
lambda b: yaml.load(b, Loader=yaml.CSafeLoader),
|
|
58
|
+
lambda v: yaml.dump(v, Dumper=yaml.CSafeDumper, allow_unicode=True),
|
|
59
|
+
)
|
|
60
|
+
return (
|
|
61
|
+
"PyYAML safe_load/safe_dump (pure Python)",
|
|
62
|
+
yaml.safe_load,
|
|
63
|
+
lambda v: yaml.safe_dump(v, allow_unicode=True),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def main() -> None:
|
|
68
|
+
ap = argparse.ArgumentParser()
|
|
69
|
+
ap.add_argument(
|
|
70
|
+
"--budget",
|
|
71
|
+
type=float,
|
|
72
|
+
default=2.0,
|
|
73
|
+
help="seconds per cell after the first 3 runs",
|
|
74
|
+
)
|
|
75
|
+
ap.add_argument("--yaml", choices=("auto", "c", "pure"), default="auto")
|
|
76
|
+
args = ap.parse_args()
|
|
77
|
+
corpus = Path(
|
|
78
|
+
os.environ.get("JARO_CORPUS") or PY_ROOT / ".." / ".." / "corpus"
|
|
79
|
+
).resolve()
|
|
80
|
+
files = sorted(corpus.glob("*.json"))
|
|
81
|
+
if not files:
|
|
82
|
+
raise SystemExit(f"no .json files in {corpus}")
|
|
83
|
+
ycodec = yaml_codec(args.yaml)
|
|
84
|
+
print(f"corpus: {corpus}")
|
|
85
|
+
print(
|
|
86
|
+
f"yaml: {ycodec[0] if ycodec else 'not installed (pip install pyyaml); YAML skipped'}"
|
|
87
|
+
)
|
|
88
|
+
print(
|
|
89
|
+
f"python: {sys.version.split()[0]} budget: {args.budget}s/cell, min {MIN_RUNS} runs, min of runs reported\n"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
results: dict[str, dict[str, dict[str, float | int]]] = {}
|
|
93
|
+
for path in files:
|
|
94
|
+
raw = path.read_bytes()
|
|
95
|
+
tree = json.loads(raw)
|
|
96
|
+
text = jaro.dumps(tree)
|
|
97
|
+
blob = jaro.dumps_binary(tree)
|
|
98
|
+
if not same(jaro.loads(text), tree) or not same(jaro.loads_binary(blob), tree):
|
|
99
|
+
raise SystemExit(f"{path.name}: round trip changed the tree")
|
|
100
|
+
text_b = text.encode("utf-8")
|
|
101
|
+
compact = json.dumps(tree, separators=(",", ":"), ensure_ascii=False)
|
|
102
|
+
cells: dict[str, tuple[Callable[[], Any], Callable[[], Any], int]] = {
|
|
103
|
+
"json": (
|
|
104
|
+
lambda: json.dumps(tree, separators=(",", ":"), ensure_ascii=False),
|
|
105
|
+
lambda: json.loads(raw),
|
|
106
|
+
len(compact.encode("utf-8")),
|
|
107
|
+
),
|
|
108
|
+
"jaro": (lambda: jaro.dumps(tree), lambda: jaro.loads(text_b), len(text_b)),
|
|
109
|
+
"jaro-bin": (
|
|
110
|
+
lambda: jaro.dumps_binary(tree),
|
|
111
|
+
lambda: jaro.loads_binary(blob),
|
|
112
|
+
len(blob),
|
|
113
|
+
),
|
|
114
|
+
}
|
|
115
|
+
if ycodec is not None:
|
|
116
|
+
ytext = ycodec[2](tree).encode("utf-8")
|
|
117
|
+
cells["yaml"] = (
|
|
118
|
+
lambda: ycodec[2](tree),
|
|
119
|
+
lambda: ycodec[1](ytext),
|
|
120
|
+
len(ytext),
|
|
121
|
+
)
|
|
122
|
+
out: dict[str, dict[str, float | int]] = {}
|
|
123
|
+
print(f"{path.name} ({len(raw):,} B source)")
|
|
124
|
+
print(f" {'format':<9}{'enc ms':>11}{'dec ms':>11}{'bytes':>12}{'vs json':>9}")
|
|
125
|
+
for name in ("json", "yaml", "jaro", "jaro-bin"):
|
|
126
|
+
if name not in cells:
|
|
127
|
+
continue
|
|
128
|
+
enc, dec, size = cells[name]
|
|
129
|
+
out[name] = {
|
|
130
|
+
"enc_ms": round(best_ms(enc, args.budget), 4),
|
|
131
|
+
"dec_ms": round(best_ms(dec, args.budget), 4),
|
|
132
|
+
"bytes": size,
|
|
133
|
+
}
|
|
134
|
+
ratio = size / cells["json"][2]
|
|
135
|
+
print(
|
|
136
|
+
f" {name:<9}{out[name]['enc_ms']:>11.3f}{out[name]['dec_ms']:>11.3f}{size:>12,}{ratio:>8.0%}"
|
|
137
|
+
)
|
|
138
|
+
results[path.name] = out
|
|
139
|
+
print()
|
|
140
|
+
(PY_ROOT / "bench" / "results.json").write_text(
|
|
141
|
+
json.dumps(results, indent=2) + "\n", "utf-8"
|
|
142
|
+
)
|
|
143
|
+
print(f"wrote {PY_ROOT / 'bench' / 'results.json'}")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
if __name__ == "__main__":
|
|
147
|
+
main()
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"canada/bin": 8.939991798251867,
|
|
3
|
+
"canada/text": 30.79489601077512,
|
|
4
|
+
"citm_catalog/bin": 11.073729256168008,
|
|
5
|
+
"citm_catalog/text": 18.012027683046956,
|
|
6
|
+
"github_issues/bin": 3.9196314666276937,
|
|
7
|
+
"github_issues/text": 8.651388828487447,
|
|
8
|
+
"npm_express/bin": 18.71837497067948,
|
|
9
|
+
"npm_express/text": 20.96664602868259,
|
|
10
|
+
"recipe_book/bin": 0.2983134450210679,
|
|
11
|
+
"recipe_book/text": 0.4587311326209807,
|
|
12
|
+
"twitter/bin": 6.971232144029012,
|
|
13
|
+
"twitter/text": 8.118465154742202
|
|
14
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Geometric-mean decode metric: mean over 12 cells of time / baseline time (lower is better).
|
|
2
|
+
|
|
3
|
+
Usage: python bench/decode_gm.py [--runs N] [--write-baseline] [--table]
|
|
4
|
+
Each cell is the per-cell minimum over N passes of decode_metric.min_ms. Prints one number
|
|
5
|
+
(the geometric mean ratio) and, on stderr, the worst cell regression. --write-baseline stores
|
|
6
|
+
the cell times in bench/decode_baseline.json.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
import sys
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
17
|
+
|
|
18
|
+
import decode_metric as dm # noqa: E402
|
|
19
|
+
|
|
20
|
+
BASELINE = Path(__file__).resolve().parent / "decode_baseline.json"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def measure(runs: int) -> dict[str, float]:
|
|
24
|
+
cells: dict[str, float] = {}
|
|
25
|
+
for _ in range(runs):
|
|
26
|
+
for p in sorted(dm.CORPUS.glob("*.json")):
|
|
27
|
+
v = json.loads(p.read_text(encoding="utf-8"))
|
|
28
|
+
text = dm.jaro.dumps(v)
|
|
29
|
+
blob = dm.jaro.dumps_binary(v)
|
|
30
|
+
for kind, fn in (
|
|
31
|
+
("text", lambda: dm.jaro.loads(text)),
|
|
32
|
+
("bin", lambda: dm.jaro.loads_binary(blob)),
|
|
33
|
+
):
|
|
34
|
+
key = f"{p.stem}/{kind}"
|
|
35
|
+
cells[key] = min(cells.get(key, math.inf), dm.min_ms(fn))
|
|
36
|
+
return cells
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def main() -> None:
|
|
40
|
+
runs = int(sys.argv[sys.argv.index("--runs") + 1]) if "--runs" in sys.argv else 1
|
|
41
|
+
cells = measure(runs)
|
|
42
|
+
if "--json" in sys.argv:
|
|
43
|
+
print(json.dumps(cells))
|
|
44
|
+
return
|
|
45
|
+
if "--write-baseline" in sys.argv:
|
|
46
|
+
BASELINE.write_text(json.dumps(cells, indent=1, sort_keys=True) + "\n")
|
|
47
|
+
print(f"wrote {BASELINE}")
|
|
48
|
+
return
|
|
49
|
+
base: dict[str, float] = json.loads(BASELINE.read_text())
|
|
50
|
+
ratios = {k: cells[k] / base[k] for k in base}
|
|
51
|
+
if "--table" in sys.argv:
|
|
52
|
+
for k, r in ratios.items():
|
|
53
|
+
print(f"{k:22s} {cells[k]:9.3f} ms ratio {r:.3f}", file=sys.stderr)
|
|
54
|
+
worst = max(ratios, key=lambda k: ratios[k])
|
|
55
|
+
print(f"worst {worst} {ratios[worst]:.3f}", file=sys.stderr)
|
|
56
|
+
print(f"{math.exp(sum(map(math.log, ratios.values())) / len(ratios)):.4f}")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
if __name__ == "__main__":
|
|
60
|
+
main()
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Decode metric: sum over the corpus files of the min decode ms, text loads + binary loads_binary.
|
|
2
|
+
|
|
3
|
+
Usage: python bench/decode_metric.py [--table]
|
|
4
|
+
Prints one number (ms, lower is better). gc is disabled around each timed loop here only,
|
|
5
|
+
to cut noise; the library is untouched. --table also prints per-file numbers on stderr.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import gc
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
PY_ROOT = Path(__file__).resolve().parent.parent
|
|
20
|
+
sys.path.insert(0, os.environ.get("JARO_SRC", str(PY_ROOT / "src")))
|
|
21
|
+
|
|
22
|
+
import jaro # noqa: E402
|
|
23
|
+
|
|
24
|
+
CORPUS = Path(os.environ.get("JARO_CORPUS", PY_ROOT.parent.parent / "corpus"))
|
|
25
|
+
RUNS = 9
|
|
26
|
+
TARGET_S = 0.05
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def min_ms(fn: Callable[[], Any]) -> float:
|
|
30
|
+
fn()
|
|
31
|
+
t0 = time.perf_counter()
|
|
32
|
+
fn()
|
|
33
|
+
one = max(time.perf_counter() - t0, 1e-6)
|
|
34
|
+
iters = max(1, round(TARGET_S / one))
|
|
35
|
+
best = float("inf")
|
|
36
|
+
for _ in range(RUNS):
|
|
37
|
+
gc.collect()
|
|
38
|
+
gc.disable()
|
|
39
|
+
try:
|
|
40
|
+
t0 = time.perf_counter()
|
|
41
|
+
for _ in range(iters):
|
|
42
|
+
fn()
|
|
43
|
+
dt = (time.perf_counter() - t0) / iters
|
|
44
|
+
finally:
|
|
45
|
+
gc.enable()
|
|
46
|
+
best = min(best, dt)
|
|
47
|
+
return best * 1000.0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def main() -> None:
|
|
51
|
+
total = 0.0
|
|
52
|
+
rows = []
|
|
53
|
+
for p in sorted(CORPUS.glob("*.json")):
|
|
54
|
+
v = json.loads(p.read_text(encoding="utf-8"))
|
|
55
|
+
text = jaro.dumps(v)
|
|
56
|
+
blob = jaro.dumps_binary(v)
|
|
57
|
+
t = min_ms(lambda: jaro.loads(text))
|
|
58
|
+
b = min_ms(lambda: jaro.loads_binary(blob))
|
|
59
|
+
rows.append((p.stem, t, b))
|
|
60
|
+
total += t + b
|
|
61
|
+
if "--table" in sys.argv:
|
|
62
|
+
for name, t, b in rows:
|
|
63
|
+
print(f"{name:15s} text {t:9.3f} bin {b:9.3f}", file=sys.stderr)
|
|
64
|
+
print(f"{total:.3f}")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
main()
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Ratio metric: JARO decode time / json.loads time per cell, geometric mean over 12 cells.
|
|
2
|
+
|
|
3
|
+
Usage: python bench/decode_ratio.py [--rounds N] [--table] [--write-baseline]
|
|
4
|
+
Each cell alternates rounds of the JARO decode and json.loads on the same file in one process and
|
|
5
|
+
keeps the minimum of each, so machine noise cancels. gc is disabled inside the timed loops only.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import gc
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
PY_ROOT = Path(__file__).resolve().parent.parent
|
|
21
|
+
sys.path.insert(0, os.environ.get("JARO_SRC", str(PY_ROOT / "src")))
|
|
22
|
+
|
|
23
|
+
import jaro # noqa: E402
|
|
24
|
+
|
|
25
|
+
CORPUS = Path(os.environ.get("JARO_CORPUS", PY_ROOT.parent.parent / "corpus"))
|
|
26
|
+
BASELINE = Path(__file__).resolve().parent / "ratio_baseline.json"
|
|
27
|
+
TARGET_S = 0.04
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def timed(fn: Callable[[], Any], iters: int) -> float:
|
|
31
|
+
gc.collect()
|
|
32
|
+
gc.disable()
|
|
33
|
+
try:
|
|
34
|
+
t0 = time.perf_counter()
|
|
35
|
+
for _ in range(iters):
|
|
36
|
+
fn()
|
|
37
|
+
return (time.perf_counter() - t0) / iters
|
|
38
|
+
finally:
|
|
39
|
+
gc.enable()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def pair(
|
|
43
|
+
a: Callable[[], Any], b: Callable[[], Any], rounds: int
|
|
44
|
+
) -> tuple[float, float]:
|
|
45
|
+
a()
|
|
46
|
+
b()
|
|
47
|
+
t0 = time.perf_counter()
|
|
48
|
+
a()
|
|
49
|
+
one = max(time.perf_counter() - t0, 1e-6)
|
|
50
|
+
ia = max(1, round(TARGET_S / one))
|
|
51
|
+
t0 = time.perf_counter()
|
|
52
|
+
b()
|
|
53
|
+
ib = max(1, round(TARGET_S / max(time.perf_counter() - t0, 1e-6)))
|
|
54
|
+
best_a = best_b = math.inf
|
|
55
|
+
for _ in range(rounds):
|
|
56
|
+
best_a = min(best_a, timed(a, ia))
|
|
57
|
+
best_b = min(best_b, timed(b, ib))
|
|
58
|
+
return best_a, best_b
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def measure(rounds: int) -> dict[str, float]:
|
|
62
|
+
ratios: dict[str, float] = {}
|
|
63
|
+
for p in sorted(CORPUS.glob("*.json")):
|
|
64
|
+
raw = p.read_text(encoding="utf-8")
|
|
65
|
+
v = json.loads(raw)
|
|
66
|
+
text = jaro.dumps(v)
|
|
67
|
+
blob = jaro.dumps_binary(v)
|
|
68
|
+
for kind, fn in (
|
|
69
|
+
("text", lambda: jaro.loads(text)),
|
|
70
|
+
("bin", lambda: jaro.loads_binary(blob)),
|
|
71
|
+
):
|
|
72
|
+
a, b = pair(fn, lambda: json.loads(raw), rounds)
|
|
73
|
+
ratios[f"{p.stem}/{kind}"] = a / b
|
|
74
|
+
return ratios
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def geomean(r: dict[str, float]) -> float:
|
|
78
|
+
return math.exp(sum(map(math.log, r.values())) / len(r))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def main() -> None:
|
|
82
|
+
rounds = (
|
|
83
|
+
int(sys.argv[sys.argv.index("--rounds") + 1]) if "--rounds" in sys.argv else 7
|
|
84
|
+
)
|
|
85
|
+
ratios = measure(rounds)
|
|
86
|
+
if "--write-baseline" in sys.argv:
|
|
87
|
+
BASELINE.write_text(json.dumps(ratios, indent=1, sort_keys=True) + "\n")
|
|
88
|
+
print(f"wrote {BASELINE} geomean {geomean(ratios):.4f}")
|
|
89
|
+
return
|
|
90
|
+
base: dict[str, float] = (
|
|
91
|
+
json.loads(BASELINE.read_text()) if BASELINE.exists() else {}
|
|
92
|
+
)
|
|
93
|
+
if "--table" in sys.argv:
|
|
94
|
+
for k, r in ratios.items():
|
|
95
|
+
extra = f" vs baseline {r / base[k]:.3f}" if k in base else ""
|
|
96
|
+
print(f"{k:22s} ratio {r:.3f}{extra}", file=sys.stderr)
|
|
97
|
+
if base:
|
|
98
|
+
worst = max(ratios, key=lambda k: ratios[k] / base[k])
|
|
99
|
+
print(
|
|
100
|
+
f"worst vs baseline {worst} {ratios[worst] / base[worst]:.3f}; geomean vs baseline {geomean(ratios) / geomean(base):.4f}",
|
|
101
|
+
file=sys.stderr,
|
|
102
|
+
)
|
|
103
|
+
print(f"{geomean(ratios):.4f}")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
if __name__ == "__main__":
|
|
107
|
+
main()
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"canada/bin": 0.5654798992407275,
|
|
3
|
+
"canada/text": 1.9396151069347145,
|
|
4
|
+
"citm_catalog/bin": 2.3570592953758154,
|
|
5
|
+
"citm_catalog/text": 3.8092739782226865,
|
|
6
|
+
"github_issues/bin": 2.1009248801303415,
|
|
7
|
+
"github_issues/text": 4.144482715105189,
|
|
8
|
+
"npm_express/bin": 4.808100671740912,
|
|
9
|
+
"npm_express/text": 6.340230721931914,
|
|
10
|
+
"recipe_book/bin": 4.7041712924118295,
|
|
11
|
+
"recipe_book/text": 7.219384139072654,
|
|
12
|
+
"twitter/bin": 2.844153703222558,
|
|
13
|
+
"twitter/text": 4.151948224942099
|
|
14
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
{
|
|
2
|
+
"canada.json": {
|
|
3
|
+
"json": {
|
|
4
|
+
"enc_ms": 27.091,
|
|
5
|
+
"dec_ms": 16.2067,
|
|
6
|
+
"bytes": 2090234
|
|
7
|
+
},
|
|
8
|
+
"yaml": {
|
|
9
|
+
"enc_ms": 247.8492,
|
|
10
|
+
"dec_ms": 305.3995,
|
|
11
|
+
"bytes": 3089399
|
|
12
|
+
},
|
|
13
|
+
"jaro": {
|
|
14
|
+
"enc_ms": 81.4531,
|
|
15
|
+
"dec_ms": 24.2975,
|
|
16
|
+
"bytes": 2090208
|
|
17
|
+
},
|
|
18
|
+
"jaro-bin": {
|
|
19
|
+
"enc_ms": 44.9303,
|
|
20
|
+
"dec_ms": 9.9295,
|
|
21
|
+
"bytes": 1112072
|
|
22
|
+
}
|
|
23
|
+
},
|
|
24
|
+
"citm_catalog.json": {
|
|
25
|
+
"json": {
|
|
26
|
+
"enc_ms": 2.1183,
|
|
27
|
+
"dec_ms": 4.627,
|
|
28
|
+
"bytes": 500299
|
|
29
|
+
},
|
|
30
|
+
"yaml": {
|
|
31
|
+
"enc_ms": 78.7743,
|
|
32
|
+
"dec_ms": 112.1735,
|
|
33
|
+
"bytes": 588331
|
|
34
|
+
},
|
|
35
|
+
"jaro": {
|
|
36
|
+
"enc_ms": 10.5187,
|
|
37
|
+
"dec_ms": 14.7006,
|
|
38
|
+
"bytes": 228077
|
|
39
|
+
},
|
|
40
|
+
"jaro-bin": {
|
|
41
|
+
"enc_ms": 16.1723,
|
|
42
|
+
"dec_ms": 9.8878,
|
|
43
|
+
"bytes": 80129
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
"github_issues.json": {
|
|
47
|
+
"json": {
|
|
48
|
+
"enc_ms": 1.7021,
|
|
49
|
+
"dec_ms": 1.6101,
|
|
50
|
+
"bytes": 586232
|
|
51
|
+
},
|
|
52
|
+
"yaml": {
|
|
53
|
+
"enc_ms": 26.3511,
|
|
54
|
+
"dec_ms": 29.2057,
|
|
55
|
+
"bytes": 611160
|
|
56
|
+
},
|
|
57
|
+
"jaro": {
|
|
58
|
+
"enc_ms": 15.8619,
|
|
59
|
+
"dec_ms": 6.1725,
|
|
60
|
+
"bytes": 489890
|
|
61
|
+
},
|
|
62
|
+
"jaro-bin": {
|
|
63
|
+
"enc_ms": 3.7143,
|
|
64
|
+
"dec_ms": 3.1443,
|
|
65
|
+
"bytes": 338660
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
"npm_express.json": {
|
|
69
|
+
"json": {
|
|
70
|
+
"enc_ms": 2.7538,
|
|
71
|
+
"dec_ms": 2.9251,
|
|
72
|
+
"bytes": 823554
|
|
73
|
+
},
|
|
74
|
+
"yaml": {
|
|
75
|
+
"enc_ms": 76.9453,
|
|
76
|
+
"dec_ms": 90.4213,
|
|
77
|
+
"bytes": 901268
|
|
78
|
+
},
|
|
79
|
+
"jaro": {
|
|
80
|
+
"enc_ms": 22.2323,
|
|
81
|
+
"dec_ms": 17.9035,
|
|
82
|
+
"bytes": 651129
|
|
83
|
+
},
|
|
84
|
+
"jaro-bin": {
|
|
85
|
+
"enc_ms": 13.8868,
|
|
86
|
+
"dec_ms": 13.8993,
|
|
87
|
+
"bytes": 288935
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
"recipe_book.json": {
|
|
91
|
+
"json": {
|
|
92
|
+
"enc_ms": 0.0644,
|
|
93
|
+
"dec_ms": 0.0606,
|
|
94
|
+
"bytes": 15705
|
|
95
|
+
},
|
|
96
|
+
"yaml": {
|
|
97
|
+
"enc_ms": 1.4042,
|
|
98
|
+
"dec_ms": 1.6185,
|
|
99
|
+
"bytes": 16450
|
|
100
|
+
},
|
|
101
|
+
"jaro": {
|
|
102
|
+
"enc_ms": 0.4675,
|
|
103
|
+
"dec_ms": 0.3439,
|
|
104
|
+
"bytes": 10919
|
|
105
|
+
},
|
|
106
|
+
"jaro-bin": {
|
|
107
|
+
"enc_ms": 0.3315,
|
|
108
|
+
"dec_ms": 0.2507,
|
|
109
|
+
"bytes": 8045
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
"twitter.json": {
|
|
113
|
+
"json": {
|
|
114
|
+
"enc_ms": 1.2749,
|
|
115
|
+
"dec_ms": 2.0924,
|
|
116
|
+
"bytes": 466906
|
|
117
|
+
},
|
|
118
|
+
"yaml": {
|
|
119
|
+
"enc_ms": 32.5884,
|
|
120
|
+
"dec_ms": 37.0575,
|
|
121
|
+
"bytes": 506544
|
|
122
|
+
},
|
|
123
|
+
"jaro": {
|
|
124
|
+
"enc_ms": 9.0975,
|
|
125
|
+
"dec_ms": 7.5778,
|
|
126
|
+
"bytes": 330143
|
|
127
|
+
},
|
|
128
|
+
"jaro-bin": {
|
|
129
|
+
"enc_ms": 5.0219,
|
|
130
|
+
"dec_ms": 5.2839,
|
|
131
|
+
"bytes": 139016
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|