macrodata-refiner 0.1.0__tar.gz → 0.2.0__tar.gz

macrodata_refiner-0.2.0/PKG-INFO

@@ -0,0 +1,151 @@
+Metadata-Version: 2.4
+Name: macrodata-refiner
+Version: 0.2.0
+Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
+Author: Macrodata Labs
+License-Expression: Apache-2.0
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: av
+Requires-Dist: cloudpickle==3.1.2
+Requires-Dist: fsspec
+Requires-Dist: httpx
+Requires-Dist: loguru
+Requires-Dist: huggingface-hub>=1.4.1
+Requires-Dist: opentelemetry-exporter-otlp-proto-http
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: numpy
+Requires-Dist: psutil
+Requires-Dist: orjson
+Requires-Dist: pyarrow
+Requires-Dist: msgspec>=0.20.0
+Requires-Dist: hf>=1.7.1
+Dynamic: license-file
+
+<p align="center">
+  <img src="https://macrodata.co/logo.svg" alt="Macrodata" width="180">
+</p>
+
+<h1 align="center">Macrodata Refiner</h1>
+
+Refiner is an open-source engine for turning raw, unstructured, and multimodal data into **high-quality datasets** for large model training.
+
+It replaces the brittle scripts and stitched-together data tooling that teams still use for training data work, while offering much better support for multimodal data, robotics workflows, and model-based processing.
+
+It also plugs into the Macrodata platform, which gives you visibility into what is happening to your data while pipelines run: job and shard lifecycle, logs, metrics, manifests, and pipeline behavior. The same code can run locally for development and then scale out through Macrodata's elastic serverless cloud.
+
+## Quickstart
+
+Install:
+
+```bash
+pip install macrodata-refiner
+```
+
+Create a Macrodata API key:
+
+- https://macrodata.co/settings/api-keys
+
+Log in:
+
+```bash
+macrodata login
+```
+
+### Cloud example
+
+Launch a robotics pipeline on Macrodata Cloud.
+
+This requires a valid API key.
+
+```python
+import refiner as mdr
+
+(
+    mdr.read_lerobot("hf://datasets/macrodata/aloha_static_battery_ep005_009")
+    .map(
+        mdr.robotics.motion_trim(
+            threshold=0.001,
+            pad_frames=5,
+        )
+    )
+    .write_lerobot("hf://buckets/macrodata/test_bucket/aloha_motion")
+    .launch_cloud(
+        name="motion_trim",
+        num_workers=4,
+    )
+)
+```
+
+### Local example
+
+Launch a local pipeline:
+
+```python
+import refiner as mdr
+
+def add_preview(row):
+    return row.update(
+        preview=" ".join(row["text"].split()[:20]),
+    )
+
+(
+    mdr.read_jsonl("input/*.jsonl")
+    .filter(mdr.col("lang") == "en")
+    .with_columns(
+        text=mdr.col("text").str.strip(),
+        text_len=mdr.col("text").str.len(),
+    )
+    .map(add_preview)
+    .write_parquet("s3://my-bucket/english-cleanup/")
+    .launch_local(
+        name="english-cleanup",
+        num_workers=2,
+    )
+)
+```
+
+`pip install` gives you:
+
+- the Python package as `refiner`
+- the CLI as `macrodata`
+
+## Batteries included
+
+- training-data-first pipeline primitives instead of generic ETL abstractions
+- multimodal processing, with robotics support today
+- a lot of built-in readers, transforms, sinks, and lifecycle/runtime machinery so you do not have to rebuild the same scaffolding in scripts
+- access to any storage backend supported by `fsspec` (S3, GCP, Hugging Face, etc.)
+- local execution for development and elastic cloud execution for large runs
+- built-in observability through the Macrodata platform, so you can inspect how your data is changing instead of debugging blindly after the fact
+
+## Docs
+
+Getting started:
+
+- [Pipeline basics](docs/pipeline-basics.md)
+- [Launchers](docs/launchers.md)
+- [CLI](docs/cli.md)
+
+Core concepts:
+
+- [Reading and writing data](docs/reading-and-writing.md)
+- [Transforms](docs/transforms.md)
+- [Expressions](docs/expressions.md)
+- [In-process debugging](docs/in-process-debugging.md)
+- [Task pipelines](docs/task-pipelines.md)
+
+Modalities and platform:
+
+- [Robotics](docs/robotics.md)
+- [Observability](docs/observability.md)
+
+## Community
+
+- join the Macrodata Discord: https://discord.gg/S8kZtmBR2x

macrodata_refiner-0.2.0/README.md

@@ -0,0 +1,121 @@
+<p align="center">
+  <img src="https://macrodata.co/logo.svg" alt="Macrodata" width="180">
+</p>
+
+<h1 align="center">Macrodata Refiner</h1>
+
+Refiner is an open-source engine for turning raw, unstructured, and multimodal data into **high-quality datasets** for large model training.
+
+It replaces the brittle scripts and stitched-together data tooling that teams still use for training data work, while offering much better support for multimodal data, robotics workflows, and model-based processing.
+
+It also plugs into the Macrodata platform, which gives you visibility into what is happening to your data while pipelines run: job and shard lifecycle, logs, metrics, manifests, and pipeline behavior. The same code can run locally for development and then scale out through Macrodata's elastic serverless cloud.
+
+## Quickstart
+
+Install:
+
+```bash
+pip install macrodata-refiner
+```
+
+Create a Macrodata API key:
+
+- https://macrodata.co/settings/api-keys
+
+Log in:
+
+```bash
+macrodata login
+```
+
+### Cloud example
+
+Launch a robotics pipeline on Macrodata Cloud.
+
+This requires a valid API key.
+
+```python
+import refiner as mdr
+
+(
+    mdr.read_lerobot("hf://datasets/macrodata/aloha_static_battery_ep005_009")
+    .map(
+        mdr.robotics.motion_trim(
+            threshold=0.001,
+            pad_frames=5,
+        )
+    )
+    .write_lerobot("hf://buckets/macrodata/test_bucket/aloha_motion")
+    .launch_cloud(
+        name="motion_trim",
+        num_workers=4,
+    )
+)
+```
+
+### Local example
+
+Launch a local pipeline:
+
+```python
+import refiner as mdr
+
+def add_preview(row):
+    return row.update(
+        preview=" ".join(row["text"].split()[:20]),
+    )
+
+(
+    mdr.read_jsonl("input/*.jsonl")
+    .filter(mdr.col("lang") == "en")
+    .with_columns(
+        text=mdr.col("text").str.strip(),
+        text_len=mdr.col("text").str.len(),
+    )
+    .map(add_preview)
+    .write_parquet("s3://my-bucket/english-cleanup/")
+    .launch_local(
+        name="english-cleanup",
+        num_workers=2,
+    )
+)
+```
+
+`pip install` gives you:
+
+- the Python package as `refiner`
+- the CLI as `macrodata`
+
+## Batteries included
+
+- training-data-first pipeline primitives instead of generic ETL abstractions
+- multimodal processing, with robotics support today
+- a lot of built-in readers, transforms, sinks, and lifecycle/runtime machinery so you do not have to rebuild the same scaffolding in scripts
+- access to any storage backend supported by `fsspec` (S3, GCP, Hugging Face, etc.)
+- local execution for development and elastic cloud execution for large runs
+- built-in observability through the Macrodata platform, so you can inspect how your data is changing instead of debugging blindly after the fact
+
+## Docs
+
+Getting started:
+
+- [Pipeline basics](docs/pipeline-basics.md)
+- [Launchers](docs/launchers.md)
+- [CLI](docs/cli.md)
+
+Core concepts:
+
+- [Reading and writing data](docs/reading-and-writing.md)
+- [Transforms](docs/transforms.md)
+- [Expressions](docs/expressions.md)
+- [In-process debugging](docs/in-process-debugging.md)
+- [Task pipelines](docs/task-pipelines.md)
+
+Modalities and platform:
+
+- [Robotics](docs/robotics.md)
+- [Observability](docs/observability.md)
+
+## Community
+
+- join the Macrodata Discord: https://discord.gg/S8kZtmBR2x

{macrodata_refiner-0.1.0 → macrodata_refiner-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "macrodata-refiner"
-version = "0.1.0"
+version = "0.2.0"
 description = "Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets"
 readme = "README.md"
 license = "Apache-2.0"
@@ -16,6 +16,7 @@ authors = [
 ]
 requires-python = ">=3.10"
 dependencies = [
+    "av",
     "cloudpickle==3.1.2",
     "fsspec",
     "httpx",
@@ -23,9 +24,12 @@ dependencies = [
     "huggingface-hub>=1.4.1",
     "opentelemetry-exporter-otlp-proto-http",
     "opentelemetry-sdk",
+    "numpy",
     "psutil",
     "orjson",
     "pyarrow",
+    "msgspec>=0.20.0",
+    "hf>=1.7.1",
 ]
 
 [project.scripts]

macrodata_refiner-0.2.0/src/macrodata_refiner.egg-info/PKG-INFO

@@ -0,0 +1,151 @@
+Metadata-Version: 2.4
+Name: macrodata-refiner
+Version: 0.2.0
+Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
+Author: Macrodata Labs
+License-Expression: Apache-2.0
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: av
+Requires-Dist: cloudpickle==3.1.2
+Requires-Dist: fsspec
+Requires-Dist: httpx
+Requires-Dist: loguru
+Requires-Dist: huggingface-hub>=1.4.1
+Requires-Dist: opentelemetry-exporter-otlp-proto-http
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: numpy
+Requires-Dist: psutil
+Requires-Dist: orjson
+Requires-Dist: pyarrow
+Requires-Dist: msgspec>=0.20.0
+Requires-Dist: hf>=1.7.1
+Dynamic: license-file
+
+<p align="center">
+  <img src="https://macrodata.co/logo.svg" alt="Macrodata" width="180">
+</p>
+
+<h1 align="center">Macrodata Refiner</h1>
+
+Refiner is an open-source engine for turning raw, unstructured, and multimodal data into **high-quality datasets** for large model training.
+
+It replaces the brittle scripts and stitched-together data tooling that teams still use for training data work, while offering much better support for multimodal data, robotics workflows, and model-based processing.
+
+It also plugs into the Macrodata platform, which gives you visibility into what is happening to your data while pipelines run: job and shard lifecycle, logs, metrics, manifests, and pipeline behavior. The same code can run locally for development and then scale out through Macrodata's elastic serverless cloud.
+
+## Quickstart
+
+Install:
+
+```bash
+pip install macrodata-refiner
+```
+
+Create a Macrodata API key:
+
+- https://macrodata.co/settings/api-keys
+
+Log in:
+
+```bash
+macrodata login
+```
+
+### Cloud example
+
+Launch a robotics pipeline on Macrodata Cloud.
+
+This requires a valid API key.
+
+```python
+import refiner as mdr
+
+(
+    mdr.read_lerobot("hf://datasets/macrodata/aloha_static_battery_ep005_009")
+    .map(
+        mdr.robotics.motion_trim(
+            threshold=0.001,
+            pad_frames=5,
+        )
+    )
+    .write_lerobot("hf://buckets/macrodata/test_bucket/aloha_motion")
+    .launch_cloud(
+        name="motion_trim",
+        num_workers=4,
+    )
+)
+```
+
+### Local example
+
+Launch a local pipeline:
+
+```python
+import refiner as mdr
+
+def add_preview(row):
+    return row.update(
+        preview=" ".join(row["text"].split()[:20]),
+    )
+
+(
+    mdr.read_jsonl("input/*.jsonl")
+    .filter(mdr.col("lang") == "en")
+    .with_columns(
+        text=mdr.col("text").str.strip(),
+        text_len=mdr.col("text").str.len(),
+    )
+    .map(add_preview)
+    .write_parquet("s3://my-bucket/english-cleanup/")
+    .launch_local(
+        name="english-cleanup",
+        num_workers=2,
+    )
+)
+```
+
+`pip install` gives you:
+
+- the Python package as `refiner`
+- the CLI as `macrodata`
+
+## Batteries included
+
+- training-data-first pipeline primitives instead of generic ETL abstractions
+- multimodal processing, with robotics support today
+- a lot of built-in readers, transforms, sinks, and lifecycle/runtime machinery so you do not have to rebuild the same scaffolding in scripts
+- access to any storage backend supported by `fsspec` (S3, GCP, Hugging Face, etc.)
+- local execution for development and elastic cloud execution for large runs
+- built-in observability through the Macrodata platform, so you can inspect how your data is changing instead of debugging blindly after the fact
+
+## Docs
+
+Getting started:
+
+- [Pipeline basics](docs/pipeline-basics.md)
+- [Launchers](docs/launchers.md)
+- [CLI](docs/cli.md)
+
+Core concepts:
+
+- [Reading and writing data](docs/reading-and-writing.md)
+- [Transforms](docs/transforms.md)
+- [Expressions](docs/expressions.md)
+- [In-process debugging](docs/in-process-debugging.md)
+- [Task pipelines](docs/task-pipelines.md)
+
+Modalities and platform:
+
+- [Robotics](docs/robotics.md)
+- [Observability](docs/observability.md)
+
+## Community
+
+- join the Macrodata Discord: https://discord.gg/S8kZtmBR2x

macrodata_refiner-0.2.0/src/macrodata_refiner.egg-info/SOURCES.txt

@@ -0,0 +1,110 @@
+LICENSE
+README.md
+pyproject.toml
+src/macrodata_refiner.egg-info/PKG-INFO
+src/macrodata_refiner.egg-info/SOURCES.txt
+src/macrodata_refiner.egg-info/dependency_links.txt
+src/macrodata_refiner.egg-info/entry_points.txt
+src/macrodata_refiner.egg-info/requires.txt
+src/macrodata_refiner.egg-info/top_level.txt
+src/refiner/__init__.py
+src/refiner/py.typed
+src/refiner/cli/__init__.py
+src/refiner/cli/auth.py
+src/refiner/cli/main.py
+src/refiner/cli/ui.py
+src/refiner/execution/__init__.py
+src/refiner/execution/buffer.py
+src/refiner/execution/engine.py
+src/refiner/execution/asyncio/runtime.py
+src/refiner/execution/asyncio/window.py
+src/refiner/execution/operators/__init__.py
+src/refiner/execution/operators/row.py
+src/refiner/execution/operators/vectorized.py
+src/refiner/execution/tracking/__init__.py
+src/refiner/execution/tracking/shards.py
+src/refiner/io/__init__.py
+src/refiner/io/datafile.py
+src/refiner/io/datafolder.py
+src/refiner/io/fileset.py
+src/refiner/launchers/__init__.py
+src/refiner/launchers/base.py
+src/refiner/launchers/cloud.py
+src/refiner/launchers/local.py
+src/refiner/media/__init__.py
+src/refiner/media/video/__init__.py
+src/refiner/media/video/remux.py
+src/refiner/media/video/transcode.py
+src/refiner/media/video/types.py
+src/refiner/media/video/writer.py
+src/refiner/pipeline/__init__.py
+src/refiner/pipeline/expressions.py
+src/refiner/pipeline/pipeline.py
+src/refiner/pipeline/planning.py
+src/refiner/pipeline/steps.py
+src/refiner/pipeline/data/block.py
+src/refiner/pipeline/data/row.py
+src/refiner/pipeline/data/shard.py
+src/refiner/pipeline/data/tabular.py
+src/refiner/pipeline/sinks/__init__.py
+src/refiner/pipeline/sinks/base.py
+src/refiner/pipeline/sinks/jsonl.py
+src/refiner/pipeline/sinks/lerobot.py
+src/refiner/pipeline/sinks/lerobot_reducer.py
+src/refiner/pipeline/sinks/parquet.py
+src/refiner/pipeline/sources/__init__.py
+src/refiner/pipeline/sources/base.py
+src/refiner/pipeline/sources/items.py
+src/refiner/pipeline/sources/task.py
+src/refiner/pipeline/sources/readers/__init__.py
+src/refiner/pipeline/sources/readers/base.py
+src/refiner/pipeline/sources/readers/csv.py
+src/refiner/pipeline/sources/readers/jsonl.py
+src/refiner/pipeline/sources/readers/lerobot.py
+src/refiner/pipeline/sources/readers/parquet.py
+src/refiner/pipeline/sources/readers/utils.py
+src/refiner/pipeline/utils/__init__.py
+src/refiner/pipeline/utils/cache/__init__.py
+src/refiner/pipeline/utils/cache/decoder_cache.py
+src/refiner/pipeline/utils/cache/file_cache.py
+src/refiner/pipeline/utils/cache/lease_cache.py
+src/refiner/platform/__init__.py
+src/refiner/platform/auth.py
+src/refiner/platform/manifest.py
+src/refiner/platform/client/__init__.py
+src/refiner/platform/client/api.py
+src/refiner/platform/client/http.py
+src/refiner/platform/client/models.py
+src/refiner/platform/client/serialize.py
+src/refiner/robotics/__init__.py
+src/refiner/robotics/motion.py
+src/refiner/robotics/lerobot_format/__init__.py
+src/refiner/robotics/lerobot_format/row.py
+src/refiner/robotics/lerobot_format/tabular.py
+src/refiner/robotics/lerobot_format/metadata/__init__.py
+src/refiner/robotics/lerobot_format/metadata/info.py
+src/refiner/robotics/lerobot_format/metadata/metadata.py
+src/refiner/robotics/lerobot_format/metadata/stats.py
+src/refiner/robotics/lerobot_format/metadata/tasks.py
+src/refiner/worker/__init__.py
+src/refiner/worker/context.py
+src/refiner/worker/entrypoint.py
+src/refiner/worker/runner.py
+src/refiner/worker/workdir.py
+src/refiner/worker/lifecycle/__init__.py
+src/refiner/worker/lifecycle/base.py
+src/refiner/worker/lifecycle/platform.py
+src/refiner/worker/lifecycle/local/__init__.py
+src/refiner/worker/lifecycle/local/claim.py
+src/refiner/worker/lifecycle/local/files.py
+src/refiner/worker/lifecycle/local/lifecycle.py
+src/refiner/worker/metrics/__init__.py
+src/refiner/worker/metrics/api.py
+src/refiner/worker/metrics/context.py
+src/refiner/worker/metrics/otel.py
+src/refiner/worker/resources/__init__.py
+src/refiner/worker/resources/cpu.py
+src/refiner/worker/resources/memory.py
+src/refiner/worker/resources/network.py
+tests/test_cache.py
+tests/test_expressions.py

{macrodata_refiner-0.1.0 → macrodata_refiner-0.2.0}/src/macrodata_refiner.egg-info/requires.txt

@@ -1,3 +1,4 @@
+av
 cloudpickle==3.1.2
 fsspec
 httpx
@@ -5,6 +6,9 @@ loguru
 huggingface-hub>=1.4.1
 opentelemetry-exporter-otlp-proto-http
 opentelemetry-sdk
+numpy
 psutil
 orjson
 pyarrow
+msgspec>=0.20.0
+hf>=1.7.1

macrodata_refiner-0.2.0/src/refiner/__init__.py

@@ -0,0 +1,57 @@
+import refiner.robotics as robotics
+from refiner.io import DataFile, DataFileSet, DataFolder
+from refiner.launchers import LaunchStats, LocalLauncher
+from refiner.media import VideoFile
+from refiner.pipeline import (
+    RefinerPipeline,
+    Row,
+    Shard,
+    from_items,
+    from_source,
+    read_csv,
+    read_jsonl,
+    read_lerobot,
+    read_parquet,
+    task,
+)
+from refiner.pipeline.expressions import coalesce, col, if_else, lit
+from refiner.worker.metrics.api import (
+    log_gauge,
+    log_gauges,
+    log_histogram,
+    log_throughput,
+    register_gauge,
+)
+from refiner.worker.runner import Worker, WorkerRunStats
+
+__all__ = [
+    "RefinerPipeline",
+    "LocalLauncher",
+    "LaunchStats",
+    "DataFile",
+    "DataFolder",
+    "DataFileSet",
+    "Shard",
+    "Row",
+    "Worker",
+    "WorkerRunStats",
+    "read_csv",
+    "read_jsonl",
+    "read_lerobot",
+    "read_parquet",
+    "from_items",
+    "from_source",
+    "task",
+    "log_throughput",
+    "log_gauge",
+    "log_gauges",
+    "register_gauge",
+    "log_histogram",
+    "col",
+    "lit",
+    "coalesce",
+    "if_else",
+    "VideoFile",
+    "Video",
+    "robotics",
+]