smithtune 0.0.1__py3-none-any.whl

smithtune/__init__.py

@@ -0,0 +1,6 @@
+"""Utilities for preparing conversational datasets."""
+
+from .splitting import split_jsonl
+
+__version__ = "0.0.1"
+__all__ = ["split_jsonl"]

smithtune/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

smithtune/cli.py

@@ -0,0 +1,32 @@
+"""Command-line interface for local dataset utilities."""
+
+import argparse
+import json
+import sys
+
+from . import __version__, split_jsonl
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="Utilities for preparing conversational datasets.")
+    parser.add_argument("--version", action="version", version=f"smithtune {__version__}")
+    commands = parser.add_subparsers(dest="command", required=True)
+    split = commands.add_parser("split", help="split a local JSONL file by group")
+    split.add_argument("input", help="input JSONL file")
+    split.add_argument("--group-by", required=True, help="top-level string or integer group field")
+    split.add_argument("--output-dir", required=True, help="new directory for the split files")
+    split.add_argument("--validation-fraction", type=float, default=0.1)
+    split.add_argument("--test-fraction", type=float, default=0.1)
+    split.add_argument("--seed", type=int, default=42)
+    args = parser.parse_args(argv)
+    try:
+        counts = split_jsonl(
+            args.input, args.output_dir, group_by=args.group_by,
+            validation_fraction=args.validation_fraction,
+            test_fraction=args.test_fraction, seed=args.seed,
+        )
+    except (ValueError, OSError) as error:
+        print(f"smithtune: {error}", file=sys.stderr)
+        return 2
+    print(json.dumps(counts))
+    return 0

smithtune/splitting.py

@@ -0,0 +1,101 @@
+"""Deterministic, grouped splitting of local JSONL files."""
+
+import hashlib
+import json
+import math
+from contextlib import ExitStack
+from pathlib import Path
+
+SPLITS = ("train", "validation", "test")
+
+
+def _object(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError("duplicate JSON key")
+        result[key] = value
+    return result
+
+
+def _reject_constant(value):
+    raise ValueError("nonstandard JSON constant")
+
+
+def split_jsonl(
+    input_path: str | Path,
+    output_dir: str | Path,
+    *,
+    group_by: str,
+    validation_fraction: float = 0.1,
+    test_fraction: float = 0.1,
+    seed: int = 42,
+) -> dict[str, int]:
+    """Split JSON objects by a top-level string or integer group ID.
+
+    Return record counts per split. Validate all input before creating output.
+    Group assignments remain stable with unchanged IDs, seed and fractions.
+    The complete input is held in memory. Existing output paths are rejected.
+    """
+    if not isinstance(group_by, str) or not group_by:
+        raise ValueError("group_by must be a nonempty field name")
+    if type(seed) is not int:
+        raise ValueError("seed must be an integer")
+    for fraction in (validation_fraction, test_fraction):
+        if (isinstance(fraction, bool) or not isinstance(fraction, (int, float))
+                or not math.isfinite(fraction) or not 0 <= fraction <= 1):
+            raise ValueError("fractions must be finite numbers between 0 and 1")
+    if validation_fraction + test_fraction > 1:
+        raise ValueError("validation and test fractions cannot total more than 1")
+
+    destination = Path(output_dir)
+    if destination.exists() or destination.is_symlink():
+        raise FileExistsError(f"output directory already exists: {destination}")
+
+    partitions: dict[str, list[bytes]] = {name: [] for name in SPLITS}
+    validation_cutoff = int(validation_fraction * 2**256)
+    test_cutoff = int((validation_fraction + test_fraction) * 2**256)
+    with Path(input_path).open("rb") as source:
+        for line_number, raw in enumerate(source, 1):
+            try:
+                record = json.loads(
+                    raw.decode("utf-8"), object_pairs_hook=_object,
+                    parse_constant=_reject_constant,
+                )
+            except (ValueError, UnicodeError, RecursionError):
+                raise ValueError(f"line {line_number}: invalid JSON object") from None
+            if not isinstance(record, dict):
+                raise ValueError(f"line {line_number}: expected a JSON object")
+            group = record.get(group_by)
+            if not (type(group) is int or isinstance(group, str) and bool(group.strip())):
+                raise ValueError(
+                    f"line {line_number}: group field must contain a nonempty string or integer"
+                )
+            identity = json.dumps([seed, group], ensure_ascii=True, separators=(",", ":"))
+            rank = int.from_bytes(hashlib.sha256(identity.encode("ascii")).digest(), "big")
+            split = "validation" if rank < validation_cutoff else (
+                "test" if rank < test_cutoff else "train"
+            )
+            partitions[split].append(raw if raw.endswith(b"\n") else raw + b"\n")
+
+    # mkdir is exclusive even if another process created the path during validation.
+    destination.mkdir()
+    created = []
+    try:
+        with ExitStack() as stack:
+            handles = {}
+            for name in SPLITS:
+                path = destination / f"{name}.jsonl"
+                handles[name] = stack.enter_context(path.open("xb"))
+                created.append(path)
+            for name, records in partitions.items():
+                handles[name].writelines(records)
+    except BaseException:
+        for path in created:
+            path.unlink(missing_ok=True)
+        try:
+            destination.rmdir()
+        except OSError:
+            pass
+        raise
+    return {name: len(records) for name, records in partitions.items()}

smithtune-0.0.1.dist-info/METADATA

@@ -0,0 +1,62 @@
+Metadata-Version: 2.4
+Name: smithtune
+Version: 0.0.1
+Summary: Utilities for preparing conversational datasets.
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# smithtune
+
+Utilities for preparing conversational datasets. Runs locally with no runtime dependencies.
+
+## Install
+
+Requires Python 3.10 or newer.
+
+```bash
+pip install smithtune
+```
+
+## Split a dataset
+
+```bash
+smithtune split data.jsonl --group-by conversation_id --output-dir splits/
+```
+
+Each input line must be a JSON object with a top-level group field containing a nonempty string or integer:
+
+```json
+{"conversation_id":"a","messages":[{"role":"user","content":"Hello"}]}
+{"conversation_id":"a","messages":[{"role":"assistant","content":"Hi"}]}
+{"conversation_id":"b","messages":[{"role":"user","content":"Good morning"}]}
+```
+
+Creates `train.jsonl`, `validation.jsonl`, and `test.jsonl` and prints record counts. Records with the same group ID always go into the same file. Use IDs that are unique across your input sources.
+
+Defaults target 80% training, 10% validation, and 10% test **by group**. Hash-based assignment gives approximate proportions; small datasets may have empty splits. Adding or reordering records does not change a group's assignment when its ID, seed, and fractions stay the same.
+
+```bash
+smithtune split data.jsonl --group-by conversation_id --output-dir splits-v2/ \
+  --validation-fraction 0.15 --test-fraction 0.15 --seed 42
+```
+
+The splitter preserves record bytes and order within each output file, adding a final newline if needed. It validates the entire input in memory before writing. Blank lines, invalid JSON, duplicate JSON keys, and missing or invalid group IDs are rejected with a line number. String IDs and integer IDs are distinct.
+
+The output directory must not exist, and its parent directory must exist. Existing files are never overwritten. No data is uploaded or fetched.
+
+## Python
+
+```python
+from smithtune import split_jsonl
+
+counts = split_jsonl("data.jsonl", "splits", group_by="conversation_id")
+```
+
+## Development
+
+```bash
+PYTHONPATH=src python -m unittest discover -s tests -v
+```

smithtune-0.0.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+smithtune/__init__.py,sha256=TNpaLUmqJv-Bv1IrhMbG0-J0XJh108j4d2gJGiAGZhc,140
+smithtune/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+smithtune/cli.py,sha256=3zmFFZJajGdipGBcrZre2Fz-V7_Vl5ZIn2ZMZyHoBsU,1378
+smithtune/splitting.py,sha256=Dq_zKWwqIpf1QUoAlwqeZGFlMkf0laauFPIj9aPcaD0,4012
+smithtune-0.0.1.dist-info/METADATA,sha256=JU1OwzDSgams2iRSb5nn-y373Ctd2xIwFSt-cM0nUbk,2246
+smithtune-0.0.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+smithtune-0.0.1.dist-info/entry_points.txt,sha256=27zgxv1bwqf-PkIt6Hs55I41huEbTtGVLhFLgWvhUes,49
+smithtune-0.0.1.dist-info/top_level.txt,sha256=b5wBjEVK75Pskj4x9_S_066AWSxXtSKkwnQfEtS0m7Q,10
+smithtune-0.0.1.dist-info/RECORD,,

smithtune-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

smithtune-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+smithtune = smithtune.cli:main

smithtune-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+smithtune