mldataforge 0.0.1__tar.gz

mldataforge-0.0.1/.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+dist
+tmp

mldataforge-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 schneiderkamplab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

mldataforge-0.0.1/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: mldataforge
+Version: 0.0.1
+Summary: swiss army knife of scripts for transforming and processing datasets for machine learning.
+Project-URL: Homepage, https://github.com/schneiderkamplab/mldataforge
+Project-URL: Bug Tracker, https://github.com/schneiderkamplab/mldataforge/issues
+Author: Peter Schneider-Kamp
+License-File: LICENSE
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.12
+Requires-Dist: click
+Requires-Dist: datasets
+Requires-Dist: mltiming
+Requires-Dist: pygz
+Description-Content-Type: text/markdown
+
+# mldatasets
+swiss army knife of scripts for transforming and processing datasets for machine learning

mldataforge-0.0.1/README.md

@@ -0,0 +1,2 @@
+# mldatasets
+swiss army knife of scripts for transforming and processing datasets for machine learning

mldataforge-0.0.1/mldataforge/__main__.py

@@ -0,0 +1,12 @@
+import click
+
+from .commands import convert
+
+@click.group()
+def cli():
+    pass
+
+cli.add_command(convert)
+
+if __name__ == "__main__":
+    cli()

mldataforge-0.0.1/mldataforge/commands/__init__.py

@@ -0,0 +1,3 @@
+from .convert import convert
+
+__all__ = ["convert"]

mldataforge-0.0.1/mldataforge/commands/convert/__init__.py

@@ -0,0 +1,11 @@
+import click
+
+from .parquet import parquet
+
+__all__ = ["convert"]
+
+@click.group()
+def convert():
+    pass
+
+convert.add_command(parquet)

mldataforge-0.0.1/mldataforge/commands/convert/parquet/__init__.py

@@ -0,0 +1,11 @@
+import click
+
+from .jsonl import jsonl
+
+__all__ = ["parquet"]
+
+@click.group()
+def parquet():
+    pass
+
+parquet.add_command(jsonl)

mldataforge-0.0.1/mldataforge/commands/convert/parquet/jsonl.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+import atexit
+import click
+from datasets import load_dataset
+from mltiming import timing
+import os
+from pygz import PigzFile
+from shutil import which
+import tempfile
+
+def create_temp_file():
+    # Create a named temp file, don't delete right away
+    temp = tempfile.NamedTemporaryFile(delete=False)
+    temp_name = temp.name
+    # Close so others can open it again without conflicts (especially on Windows)
+    temp.close()
+
+    # Schedule its deletion at exit
+    atexit.register(_cleanup_file, temp_name)
+
+    return temp_name
+
+def _cleanup_file(file_path):
+    try:
+        os.remove(file_path)
+    except OSError:
+        pass
+
+@click.command()
+@click.argument("parquet_file", type=click.Path(exists=True))
+@click.argument("output_file", type=click.Path(exists=False), required=False)
+@click.option("--compression", default="infer", type=click.Choice(["none", "infer", "pigz", "gzip", "bz2", "xz"]), help="Compress the output JSONL file (default: infer; pigz for parallel gzip).")
+@click.option("--threads", default=64, help="Number of processes to use for pigz compression (default: 64).")
+@click.option("--overwrite", is_flag=True, help="Overwrite existing JSONL files.")
+def jsonl(parquet_file, output_file, compression, threads, overwrite):
+    if os.path.exists(output_file) and not overwrite:
+        raise click.ClickException(f"Output file {output_file} already exists. Use --overwrite to overwrite.")
+    with timing(message=f"Loading from {parquet_file}"):
+        ds = load_dataset("parquet", data_files=parquet_file)
+    orig_output_file = None
+    if compression == "none":
+        compression = None
+    elif compression == "infer":
+        if output_file.endswith(".gz") and which("pigz") is not None:
+            compression = "pigz"
+    if compression == "pigz":
+        compression = None
+        orig_output_file = output_file
+        output_file = create_temp_file()
+    with timing(message=f"Saving to {output_file} with compression {compression}"):
+        ds["train"].to_json(output_file, orient="records", lines=True, compression=compression)
+    if orig_output_file is not None:
+        with timing(message=f"Compressing {output_file} to {orig_output_file} with pigz using {threads} threads"):
+            with open(output_file, "rt") as f_in, PigzFile(f"{orig_output_file}.gz", "wt", threads=threads) as f_out:
+                f_out.write(f_in.read())

mldataforge-0.0.1/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "mldataforge"
+version = "0.0.1"
+authors = [
+  { name = "Peter Schneider-Kamp" }
+]
+
+description = "swiss army knife of scripts for transforming and processing datasets for machine learning."
+readme = "README.md"
+requires-python = ">=3.12"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+
+dependencies = [
+    'click',
+    'datasets',
+    'mltiming',
+    'pygz'
+]
+
+[project.urls]
+"Homepage" = "https://github.com/schneiderkamplab/mldataforge"
+"Bug Tracker" = "https://github.com/schneiderkamplab/mldataforge/issues"
+
+[tool.hatch.build]
+include = [
+    'mldataforge',
+    'LICENSE',
+    'README.md',
+]