smithtune 0.0.1__tar.gz

smithtune-0.0.1/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.4
+Name: smithtune
+Version: 0.0.1
+Summary: Utilities for preparing conversational datasets.
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# smithtune
+
+Utilities for preparing conversational datasets. Runs locally with no runtime dependencies.
+
+## Install
+
+Requires Python 3.10 or newer.
+
+```bash
+pip install smithtune
+```
+
+## Split a dataset
+
+```bash
+smithtune split data.jsonl --group-by conversation_id --output-dir splits/
+```
+
+Each input line must be a JSON object with a top-level group field containing a nonempty string or integer:
+
+```json
+{"conversation_id":"a","messages":[{"role":"user","content":"Hello"}]}
+{"conversation_id":"a","messages":[{"role":"assistant","content":"Hi"}]}
+{"conversation_id":"b","messages":[{"role":"user","content":"Good morning"}]}
+```
+
+Creates `train.jsonl`, `validation.jsonl`, and `test.jsonl` and prints record counts. Records with the same group ID always go into the same file. Use IDs that are unique across your input sources.
+
+Defaults target 80% training, 10% validation, and 10% test **by group**. Hash-based assignment gives approximate proportions; small datasets may have empty splits. Adding or reordering records does not change a group's assignment when its ID, seed, and fractions stay the same.
+
+```bash
+smithtune split data.jsonl --group-by conversation_id --output-dir splits-v2/ \
+  --validation-fraction 0.15 --test-fraction 0.15 --seed 42
+```
+
+The splitter preserves record bytes and order within each output file, adding a final newline if needed. It validates the entire input in memory before writing. Blank lines, invalid JSON, duplicate JSON keys, and missing or invalid group IDs are rejected with a line number. String IDs and integer IDs are distinct.
+
+The output directory must not exist, and its parent directory must exist. Existing files are never overwritten. No data is uploaded or fetched.
+
+## Python
+
+```python
+from smithtune import split_jsonl
+
+counts = split_jsonl("data.jsonl", "splits", group_by="conversation_id")
+```
+
+## Development
+
+```bash
+PYTHONPATH=src python -m unittest discover -s tests -v
+```

smithtune-0.0.1/README.md

@@ -0,0 +1,52 @@
+# smithtune
+
+Utilities for preparing conversational datasets. Runs locally with no runtime dependencies.
+
+## Install
+
+Requires Python 3.10 or newer.
+
+```bash
+pip install smithtune
+```
+
+## Split a dataset
+
+```bash
+smithtune split data.jsonl --group-by conversation_id --output-dir splits/
+```
+
+Each input line must be a JSON object with a top-level group field containing a nonempty string or integer:
+
+```json
+{"conversation_id":"a","messages":[{"role":"user","content":"Hello"}]}
+{"conversation_id":"a","messages":[{"role":"assistant","content":"Hi"}]}
+{"conversation_id":"b","messages":[{"role":"user","content":"Good morning"}]}
+```
+
+Creates `train.jsonl`, `validation.jsonl`, and `test.jsonl` and prints record counts. Records with the same group ID always go into the same file. Use IDs that are unique across your input sources.
+
+Defaults target 80% training, 10% validation, and 10% test **by group**. Hash-based assignment gives approximate proportions; small datasets may have empty splits. Adding or reordering records does not change a group's assignment when its ID, seed, and fractions stay the same.
+
+```bash
+smithtune split data.jsonl --group-by conversation_id --output-dir splits-v2/ \
+  --validation-fraction 0.15 --test-fraction 0.15 --seed 42
+```
+
+The splitter preserves record bytes and order within each output file, adding a final newline if needed. It validates the entire input in memory before writing. Blank lines, invalid JSON, duplicate JSON keys, and missing or invalid group IDs are rejected with a line number. String IDs and integer IDs are distinct.
+
+The output directory must not exist, and its parent directory must exist. Existing files are never overwritten. No data is uploaded or fetched.
+
+## Python
+
+```python
+from smithtune import split_jsonl
+
+counts = split_jsonl("data.jsonl", "splits", group_by="conversation_id")
+```
+
+## Development
+
+```bash
+PYTHONPATH=src python -m unittest discover -s tests -v
+```

smithtune-0.0.1/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["setuptools>=77,<85"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "smithtune"
+version = "0.0.1"
+description = "Utilities for preparing conversational datasets."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = []
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Programming Language :: Python :: 3",
+  "Topic :: Utilities",
+]
+
+[project.scripts]
+smithtune = "smithtune.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["smithtune*"]
+
+[tool.setuptools]
+include-package-data = false

smithtune-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

smithtune-0.0.1/src/smithtune/__init__.py

@@ -0,0 +1,6 @@
+"""Utilities for preparing conversational datasets."""
+
+from .splitting import split_jsonl
+
+__version__ = "0.0.1"
+__all__ = ["split_jsonl"]

smithtune-0.0.1/src/smithtune/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

smithtune-0.0.1/src/smithtune/cli.py

@@ -0,0 +1,32 @@
+"""Command-line interface for local dataset utilities."""
+
+import argparse
+import json
+import sys
+
+from . import __version__, split_jsonl
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="Utilities for preparing conversational datasets.")
+    parser.add_argument("--version", action="version", version=f"smithtune {__version__}")
+    commands = parser.add_subparsers(dest="command", required=True)
+    split = commands.add_parser("split", help="split a local JSONL file by group")
+    split.add_argument("input", help="input JSONL file")
+    split.add_argument("--group-by", required=True, help="top-level string or integer group field")
+    split.add_argument("--output-dir", required=True, help="new directory for the split files")
+    split.add_argument("--validation-fraction", type=float, default=0.1)
+    split.add_argument("--test-fraction", type=float, default=0.1)
+    split.add_argument("--seed", type=int, default=42)
+    args = parser.parse_args(argv)
+    try:
+        counts = split_jsonl(
+            args.input, args.output_dir, group_by=args.group_by,
+            validation_fraction=args.validation_fraction,
+            test_fraction=args.test_fraction, seed=args.seed,
+        )
+    except (ValueError, OSError) as error:
+        print(f"smithtune: {error}", file=sys.stderr)
+        return 2
+    print(json.dumps(counts))
+    return 0

smithtune-0.0.1/src/smithtune/splitting.py

@@ -0,0 +1,101 @@
+"""Deterministic, grouped splitting of local JSONL files."""
+
+import hashlib
+import json
+import math
+from contextlib import ExitStack
+from pathlib import Path
+
+SPLITS = ("train", "validation", "test")
+
+
+def _object(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError("duplicate JSON key")
+        result[key] = value
+    return result
+
+
+def _reject_constant(value):
+    raise ValueError("nonstandard JSON constant")
+
+
+def split_jsonl(
+    input_path: str | Path,
+    output_dir: str | Path,
+    *,
+    group_by: str,
+    validation_fraction: float = 0.1,
+    test_fraction: float = 0.1,
+    seed: int = 42,
+) -> dict[str, int]:
+    """Split JSON objects by a top-level string or integer group ID.
+
+    Return record counts per split. Validate all input before creating output.
+    Group assignments remain stable with unchanged IDs, seed and fractions.
+    The complete input is held in memory. Existing output paths are rejected.
+    """
+    if not isinstance(group_by, str) or not group_by:
+        raise ValueError("group_by must be a nonempty field name")
+    if type(seed) is not int:
+        raise ValueError("seed must be an integer")
+    for fraction in (validation_fraction, test_fraction):
+        if (isinstance(fraction, bool) or not isinstance(fraction, (int, float))
+                or not math.isfinite(fraction) or not 0 <= fraction <= 1):
+            raise ValueError("fractions must be finite numbers between 0 and 1")
+    if validation_fraction + test_fraction > 1:
+        raise ValueError("validation and test fractions cannot total more than 1")
+
+    destination = Path(output_dir)
+    if destination.exists() or destination.is_symlink():
+        raise FileExistsError(f"output directory already exists: {destination}")
+
+    partitions: dict[str, list[bytes]] = {name: [] for name in SPLITS}
+    validation_cutoff = int(validation_fraction * 2**256)
+    test_cutoff = int((validation_fraction + test_fraction) * 2**256)
+    with Path(input_path).open("rb") as source:
+        for line_number, raw in enumerate(source, 1):
+            try:
+                record = json.loads(
+                    raw.decode("utf-8"), object_pairs_hook=_object,
+                    parse_constant=_reject_constant,
+                )
+            except (ValueError, UnicodeError, RecursionError):
+                raise ValueError(f"line {line_number}: invalid JSON object") from None
+            if not isinstance(record, dict):
+                raise ValueError(f"line {line_number}: expected a JSON object")
+            group = record.get(group_by)
+            if not (type(group) is int or isinstance(group, str) and bool(group.strip())):
+                raise ValueError(
+                    f"line {line_number}: group field must contain a nonempty string or integer"
+                )
+            identity = json.dumps([seed, group], ensure_ascii=True, separators=(",", ":"))
+            rank = int.from_bytes(hashlib.sha256(identity.encode("ascii")).digest(), "big")
+            split = "validation" if rank < validation_cutoff else (
+                "test" if rank < test_cutoff else "train"
+            )
+            partitions[split].append(raw if raw.endswith(b"\n") else raw + b"\n")
+
+    # mkdir is exclusive even if another process created the path during validation.
+    destination.mkdir()
+    created = []
+    try:
+        with ExitStack() as stack:
+            handles = {}
+            for name in SPLITS:
+                path = destination / f"{name}.jsonl"
+                handles[name] = stack.enter_context(path.open("xb"))
+                created.append(path)
+            for name, records in partitions.items():
+                handles[name].writelines(records)
+    except BaseException:
+        for path in created:
+            path.unlink(missing_ok=True)
+        try:
+            destination.rmdir()
+        except OSError:
+            pass
+        raise
+    return {name: len(records) for name, records in partitions.items()}

smithtune-0.0.1/src/smithtune.egg-info/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.4
+Name: smithtune
+Version: 0.0.1
+Summary: Utilities for preparing conversational datasets.
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# smithtune
+
+Utilities for preparing conversational datasets. Runs locally with no runtime dependencies.
+
+## Install
+
+Requires Python 3.10 or newer.
+
+```bash
+pip install smithtune
+```
+
+## Split a dataset
+
+```bash
+smithtune split data.jsonl --group-by conversation_id --output-dir splits/
+```
+
+Each input line must be a JSON object with a top-level group field containing a nonempty string or integer:
+
+```json
+{"conversation_id":"a","messages":[{"role":"user","content":"Hello"}]}
+{"conversation_id":"a","messages":[{"role":"assistant","content":"Hi"}]}
+{"conversation_id":"b","messages":[{"role":"user","content":"Good morning"}]}
+```
+
+Creates `train.jsonl`, `validation.jsonl`, and `test.jsonl` and prints record counts. Records with the same group ID always go into the same file. Use IDs that are unique across your input sources.
+
+Defaults target 80% training, 10% validation, and 10% test **by group**. Hash-based assignment gives approximate proportions; small datasets may have empty splits. Adding or reordering records does not change a group's assignment when its ID, seed, and fractions stay the same.
+
+```bash
+smithtune split data.jsonl --group-by conversation_id --output-dir splits-v2/ \
+  --validation-fraction 0.15 --test-fraction 0.15 --seed 42
+```
+
+The splitter preserves record bytes and order within each output file, adding a final newline if needed. It validates the entire input in memory before writing. Blank lines, invalid JSON, duplicate JSON keys, and missing or invalid group IDs are rejected with a line number. String IDs and integer IDs are distinct.
+
+The output directory must not exist, and its parent directory must exist. Existing files are never overwritten. No data is uploaded or fetched.
+
+## Python
+
+```python
+from smithtune import split_jsonl
+
+counts = split_jsonl("data.jsonl", "splits", group_by="conversation_id")
+```
+
+## Development
+
+```bash
+PYTHONPATH=src python -m unittest discover -s tests -v
+```

smithtune-0.0.1/src/smithtune.egg-info/SOURCES.txt

@@ -0,0 +1,12 @@
+README.md
+pyproject.toml
+src/smithtune/__init__.py
+src/smithtune/__main__.py
+src/smithtune/cli.py
+src/smithtune/splitting.py
+src/smithtune.egg-info/PKG-INFO
+src/smithtune.egg-info/SOURCES.txt
+src/smithtune.egg-info/dependency_links.txt
+src/smithtune.egg-info/entry_points.txt
+src/smithtune.egg-info/top_level.txt
+tests/test_splitting.py

smithtune-0.0.1/src/smithtune.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

smithtune-0.0.1/src/smithtune.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+smithtune = smithtune.cli:main

smithtune-0.0.1/src/smithtune.egg-info/top_level.txt

@@ -0,0 +1 @@
+smithtune

smithtune-0.0.1/tests/test_splitting.py

@@ -0,0 +1,163 @@
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from smithtune import split_jsonl
+
+
+class SplitTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.source = self.root / "data.jsonl"
+        self.output = self.root / "splits"
+
+    def write_rows(self, rows):
+        self.source.write_text("".join(json.dumps(row) + "\n" for row in rows))
+
+    def run_split(self, **kwargs):
+        return split_jsonl(self.source, self.output, group_by="conversation_id", **kwargs)
+
+    def assignments(self, directory):
+        memberships = {}
+        for file in directory.glob("*.jsonl"):
+            for line in file.read_text().splitlines():
+                group = json.loads(line)["conversation_id"]
+                key = (type(group), group)
+                self.assertEqual(memberships.setdefault(key, file.stem), file.stem)
+        return memberships
+
+    def test_groups_stay_together_and_all_records_survive(self):
+        rows = [{"conversation_id": str(i), "turn": turn} for i in range(300) for turn in range(3)]
+        self.write_rows(rows)
+        counts = self.run_split()
+        self.assertEqual(sum(counts.values()), len(rows))
+        self.assertTrue(all(count > 0 for count in counts.values()))
+        self.assertEqual(len(self.assignments(self.output)), 300)
+        actual = [json.loads(line) for path in self.output.glob("*.jsonl") for line in path.read_text().splitlines()]
+        self.assertCountEqual(actual, rows)
+
+    def test_membership_survives_append_and_reordering(self):
+        rows = [{"conversation_id": str(i)} for i in range(100)]
+        self.write_rows(rows)
+        self.run_split()
+        before = self.assignments(self.output)
+        self.output = self.root / "second"
+        self.write_rows(list(reversed(rows)) + [{"conversation_id": "new"}])
+        self.run_split()
+        after = self.assignments(self.output)
+        self.assertTrue(all(after[key] == value for key, value in before.items()))
+
+    def test_exact_record_bytes_and_order_preserved(self):
+        raw = b'{ "conversation_id": "a", "text": "caf\\u00e9" }\r\n{"conversation_id":"a","n":1.00}'
+        self.source.write_bytes(raw)
+        self.run_split(validation_fraction=0, test_fraction=0)
+        self.assertEqual((self.output / "train.jsonl").read_bytes(), raw + b"\n")
+
+    def test_invalid_records_create_no_output(self):
+        invalid = [b"\n", b"{bad}\n", b"[]\n", b"{}\n", b'{"conversation_id":null}\n',
+                   b'{"conversation_id":true}\n', b'{"conversation_id":1.5}\n',
+                   b'{"conversation_id":" "}\n', b'{"conversation_id":[]}\n',
+                   b'{"conversation_id":"a","x":NaN}\n',
+                   b'{"conversation_id":"a","conversation_id":"b"}\n',
+                   b'{"conversation_id":"a","text":"\xff"}\n']
+        for raw in invalid:
+            with self.subTest(raw=raw):
+                self.source.write_bytes(b'{"conversation_id":"valid"}\n' + raw)
+                with self.assertRaisesRegex(ValueError, "line 2"):
+                    self.run_split()
+                self.assertFalse(self.output.exists())
+
+    def test_existing_directory_is_untouched(self):
+        self.write_rows([{"conversation_id": "a"}])
+        self.output.mkdir()
+        marker = self.output / "train.jsonl"
+        marker.write_bytes(b"keep me")
+        with self.assertRaises(FileExistsError):
+            self.run_split()
+        self.assertEqual(marker.read_bytes(), b"keep me")
+
+    def test_output_cannot_be_input_file(self):
+        self.write_rows([{"conversation_id": "a"}])
+        before = self.source.read_bytes()
+        self.output = self.source
+        with self.assertRaises(FileExistsError):
+            self.run_split()
+        self.assertEqual(self.source.read_bytes(), before)
+
+    def test_dangling_symlink_rejected(self):
+        self.write_rows([])
+        self.output.symlink_to(self.root / "missing")
+        with self.assertRaises(FileExistsError):
+            self.run_split()
+        self.assertTrue(self.output.is_symlink())
+
+    def test_invalid_fractions_rejected(self):
+        self.write_rows([])
+        for fraction in [-0.1, 1.1, float("nan"), float("inf"), True, "0.1"]:
+            with self.subTest(fraction=fraction):
+                with self.assertRaises(ValueError):
+                    self.run_split(validation_fraction=fraction)
+        with self.assertRaises(ValueError):
+            self.run_split(validation_fraction=0.7, test_fraction=0.4)
+        self.assertFalse(self.output.exists())
+
+    def test_zero_and_one_fractions(self):
+        self.write_rows([{"conversation_id": i} for i in range(10)])
+        counts = self.run_split(validation_fraction=1, test_fraction=0)
+        self.assertEqual(counts, {"train": 0, "validation": 10, "test": 0})
+
+    def test_empty_input(self):
+        self.write_rows([])
+        self.assertEqual(self.run_split(), {"train": 0, "validation": 0, "test": 0})
+        self.assertEqual(len(list(self.output.glob("*.jsonl"))), 3)
+
+    def test_string_and_integer_ids_are_distinct(self):
+        self.write_rows([{"conversation_id": i} for i in range(100)] +
+                        [{"conversation_id": str(i)} for i in range(100)])
+        self.run_split()
+        memberships = self.assignments(self.output)
+        self.assertTrue(any(memberships[int, i] != memberships[str, str(i)] for i in range(100)))
+
+    def test_invalid_group_field_and_seed(self):
+        self.write_rows([])
+        for kwargs in [{"group_by": ""}, {"group_by": None}, {"group_by": "id", "seed": True}]:
+            with self.assertRaises(ValueError):
+                split_jsonl(self.source, self.output, **kwargs)
+
+    def test_failed_output_creation_cleans_up_own_files(self):
+        self.write_rows([{"conversation_id": "a"}])
+        original = Path.open
+
+        def fail_second(path, *args, **kwargs):
+            if path.name == "validation.jsonl":
+                raise OSError("simulated full disk")
+            return original(path, *args, **kwargs)
+
+        with patch.object(Path, "open", fail_second):
+            with self.assertRaisesRegex(OSError, "simulated full disk"):
+                self.run_split()
+        self.assertFalse(self.output.exists())
+
+    def test_cli_reports_counts_and_readable_error(self):
+        self.write_rows([{"conversation_id": "a"}])
+        env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[1] / "src")}
+        command = [sys.executable, "-m", "smithtune", "split", str(self.source),
+                   "--group-by", "conversation_id", "--output-dir", str(self.output)]
+        result = subprocess.run(command, capture_output=True, text=True, env=env)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(sum(json.loads(result.stdout).values()), 1)
+        again = subprocess.run(command, capture_output=True, text=True, env=env)
+        self.assertEqual(again.returncode, 2)
+        self.assertIn("already exists", again.stderr)
+        self.assertNotIn("Traceback", again.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()