macrodata-refiner 0.2.2__tar.gz → 0.3.0__tar.gz

{macrodata_refiner-0.2.2 → macrodata_refiner-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.2.2
+Version: 0.3.0
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -13,30 +13,55 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: cloudpickle==3.1.2
-Requires-Dist: fsspec
+Requires-Dist: fsspec[http]
 Requires-Dist: httpx
 Requires-Dist: loguru
-Requires-Dist: opentelemetry-exporter-otlp-proto-http
-Requires-Dist: opentelemetry-sdk
 Requires-Dist: numpy
-Requires-Dist: psutil
 Requires-Dist: orjson
+Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: msgspec>=0.20.0
+Requires-Dist: pydantic>=2.0.0
 Provides-Extra: video
 Requires-Dist: av; extra == "video"
-Provides-Extra: robotics
-Requires-Dist: macrodata-refiner[video]; extra == "robotics"
-Requires-Dist: huggingface-hub>=1.4.1; extra == "robotics"
-Requires-Dist: hf>=1.7.1; extra == "robotics"
+Requires-Dist: pillow; extra == "video"
+Provides-Extra: hf
+Requires-Dist: datasets>=3.0.0; extra == "hf"
+Requires-Dist: huggingface-hub>=1.4.1; extra == "hf"
+Requires-Dist: hf>=1.7.1; extra == "hf"
+Provides-Extra: egocentric
+Requires-Dist: macrodata-refiner[hf]; extra == "egocentric"
+Requires-Dist: macrodata-refiner[video]; extra == "egocentric"
+Requires-Dist: ego-vision[models]>=0.1.8; extra == "egocentric"
 Provides-Extra: text
 Requires-Dist: warcio; extra == "text"
+Provides-Extra: hdf5
+Requires-Dist: h5py; extra == "hdf5"
+Provides-Extra: zarr
+Requires-Dist: zarr<3,>=2.18; extra == "zarr"
+Requires-Dist: numcodecs<0.16; extra == "zarr"
+Provides-Extra: mcap
+Requires-Dist: av; extra == "mcap"
+Requires-Dist: mcap; extra == "mcap"
+Requires-Dist: mcap-protobuf-support; extra == "mcap"
+Requires-Dist: mcap-ros2-support; extra == "mcap"
+Requires-Dist: pillow; extra == "mcap"
 Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
+Provides-Extra: tensorflow
+Requires-Dist: tensorflow; extra == "tensorflow"
+Provides-Extra: tfds
+Requires-Dist: macrodata-refiner[tensorflow]; extra == "tfds"
+Requires-Dist: tensorflow-datasets; extra == "tfds"
 Provides-Extra: testing
-Requires-Dist: macrodata-refiner[robotics]; extra == "testing"
+Requires-Dist: macrodata-refiner[hdf5]; extra == "testing"
+Requires-Dist: macrodata-refiner[hf]; extra == "testing"
+Requires-Dist: macrodata-refiner[mcap]; extra == "testing"
+Requires-Dist: macrodata-refiner[video]; extra == "testing"
+Requires-Dist: macrodata-refiner[zarr]; extra == "testing"
 Requires-Dist: macrodata-refiner[text]; extra == "testing"
 Requires-Dist: macrodata-refiner[s3]; extra == "testing"
+Requires-Dist: macrodata-refiner[tfds]; extra == "testing"
 Requires-Dist: pytest>=8.0.0; extra == "testing"
 Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
 Provides-Extra: all
@@ -49,9 +74,10 @@ Dynamic: license-file
 
 <h1 align="center">Macrodata Refiner</h1>
 
-Refiner is an open-source engine for turning raw, unstructured, and multimodal data into **high-quality datasets** for large model training.
+Refiner is an open-source engine for turning raw robotics and multimodal data into **high-quality datasets** for model training.
 
-It replaces the brittle scripts and stitched-together data tooling that teams still use for training data work, while offering much better support for multimodal data, robotics workflows, and model-based processing.
+It gives training-data teams one pipeline model for multimodal data, robotics
+workflows, and model-based processing.
 
 It also plugs into the Macrodata platform, which gives you visibility into what is happening to your data while pipelines run: job and shard lifecycle, logs, metrics, manifests, and pipeline behavior. The same code can run locally for development and then scale out through Macrodata's elastic serverless cloud.
 
@@ -90,7 +116,7 @@ import refiner as mdr
             pad_frames=5,
         )
     )
-    .write_lerobot("hf://buckets/macrodata/test_bucket/aloha_motion")
+    .write_lerobot("hf://buckets/acme-robotics/aloha_motion")
     .launch_cloud(
         name="motion_trim",
         num_workers=4,
@@ -98,7 +124,7 @@ import refiner as mdr
 )
 ```
 
-Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+Need cloud GPUs? See [Resources, GPUs, and Services](docs/running-pipelines/resources-gpus-and-services.md).
 
 ### Local example
 
@@ -137,31 +163,33 @@ def add_preview(row):
 
 - training-data-first pipeline primitives instead of generic ETL abstractions
 - multimodal processing, with robotics support today
-- a lot of built-in readers, transforms, sinks, and lifecycle/runtime machinery so you do not have to rebuild the same scaffolding in scripts
+- built-in readers, transforms, sinks, and runtime machinery for common dataset work
 - access to any storage backend supported by `fsspec` (S3, GCP, Hugging Face, etc.)
 - local execution for development and elastic cloud execution for large runs
-- built-in observability through the Macrodata platform, so you can inspect how your data is changing instead of debugging blindly after the fact
+- built-in observability through the Macrodata platform for job state, logs, metrics, and manifests
 
 ## Docs
 
-Getting started:
+Start here:
 
-- [Pipeline basics](docs/pipeline-basics.md)
-- [Launchers](docs/launchers.md)
-- [CLI](docs/cli.md)
+- [Docs index](docs/index.md)
+- [Quickstart](docs/quickstart.md)
+- [Running pipelines](docs/running-pipelines/index.md)
 
-Core concepts:
+Build a dataset:
 
-- [Reading and writing data](docs/reading-and-writing.md)
-- [Transforms](docs/transforms.md)
-- [Expressions](docs/expressions.md)
-- [In-process debugging](docs/in-process-debugging.md)
-- [Task pipelines](docs/task-pipelines.md)
+- [Reading data](docs/reading-data/index.md)
+- [Episode data](docs/episode-data/index.md)
+- [Transforms](docs/transforms/index.md)
+- [Episode operations](docs/episode-operations/index.md)
+- [Writing data](docs/writing-data/index.md)
+- [Examples](docs/examples/index.md)
 
-Modalities and platform:
+Operate jobs:
 
-- [Robotics](docs/robotics.md)
-- [Observability](docs/observability.md)
+- [Platform](docs/platform/index.md)
+- [CLI](docs/cli/index.md)
+- [Reference](docs/reference/index.md)
 
 ## Community
 

{macrodata_refiner-0.2.2 → macrodata_refiner-0.3.0}/README.md

@@ -4,9 +4,10 @@
 
 <h1 align="center">Macrodata Refiner</h1>
 
-Refiner is an open-source engine for turning raw, unstructured, and multimodal data into **high-quality datasets** for large model training.
+Refiner is an open-source engine for turning raw robotics and multimodal data into **high-quality datasets** for model training.
 
-It replaces the brittle scripts and stitched-together data tooling that teams still use for training data work, while offering much better support for multimodal data, robotics workflows, and model-based processing.
+It gives training-data teams one pipeline model for multimodal data, robotics
+workflows, and model-based processing.
 
 It also plugs into the Macrodata platform, which gives you visibility into what is happening to your data while pipelines run: job and shard lifecycle, logs, metrics, manifests, and pipeline behavior. The same code can run locally for development and then scale out through Macrodata's elastic serverless cloud.
 
@@ -45,7 +46,7 @@ import refiner as mdr
             pad_frames=5,
         )
     )
-    .write_lerobot("hf://buckets/macrodata/test_bucket/aloha_motion")
+    .write_lerobot("hf://buckets/acme-robotics/aloha_motion")
     .launch_cloud(
         name="motion_trim",
         num_workers=4,
@@ -53,7 +54,7 @@ import refiner as mdr
 )
 ```
 
-Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+Need cloud GPUs? See [Resources, GPUs, and Services](docs/running-pipelines/resources-gpus-and-services.md).
 
 ### Local example
 
@@ -92,31 +93,33 @@ def add_preview(row):
 
 - training-data-first pipeline primitives instead of generic ETL abstractions
 - multimodal processing, with robotics support today
-- a lot of built-in readers, transforms, sinks, and lifecycle/runtime machinery so you do not have to rebuild the same scaffolding in scripts
+- built-in readers, transforms, sinks, and runtime machinery for common dataset work
 - access to any storage backend supported by `fsspec` (S3, GCP, Hugging Face, etc.)
 - local execution for development and elastic cloud execution for large runs
-- built-in observability through the Macrodata platform, so you can inspect how your data is changing instead of debugging blindly after the fact
+- built-in observability through the Macrodata platform for job state, logs, metrics, and manifests
 
 ## Docs
 
-Getting started:
+Start here:
 
-- [Pipeline basics](docs/pipeline-basics.md)
-- [Launchers](docs/launchers.md)
-- [CLI](docs/cli.md)
+- [Docs index](docs/index.md)
+- [Quickstart](docs/quickstart.md)
+- [Running pipelines](docs/running-pipelines/index.md)
 
-Core concepts:
+Build a dataset:
 
-- [Reading and writing data](docs/reading-and-writing.md)
-- [Transforms](docs/transforms.md)
-- [Expressions](docs/expressions.md)
-- [In-process debugging](docs/in-process-debugging.md)
-- [Task pipelines](docs/task-pipelines.md)
+- [Reading data](docs/reading-data/index.md)
+- [Episode data](docs/episode-data/index.md)
+- [Transforms](docs/transforms/index.md)
+- [Episode operations](docs/episode-operations/index.md)
+- [Writing data](docs/writing-data/index.md)
+- [Examples](docs/examples/index.md)
 
-Modalities and platform:
+Operate jobs:
 
-- [Robotics](docs/robotics.md)
-- [Observability](docs/observability.md)
+- [Platform](docs/platform/index.md)
+- [CLI](docs/cli/index.md)
+- [Reference](docs/reference/index.md)
 
 ## Community
 

{macrodata_refiner-0.2.2 → macrodata_refiner-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "macrodata-refiner"
-version = "0.2.2"
+version = "0.3.0"
 description = "Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets"
 readme = "README.md"
 license = "Apache-2.0"
@@ -17,37 +17,68 @@ authors = [
 requires-python = ">=3.10"
 dependencies = [
     "cloudpickle==3.1.2",
-    "fsspec",
+    "fsspec[http]",
     "httpx",
     "loguru",
-    "opentelemetry-exporter-otlp-proto-http",
-    "opentelemetry-sdk",
     "numpy",
-    "psutil",
     "orjson",
+    "packaging",
     "pyarrow",
     "msgspec>=0.20.0",
+    "pydantic>=2.0.0",
 ]
 
 [project.optional-dependencies]
 video = [
     "av",
+    "pillow",
 ]
-robotics = [
-    "macrodata-refiner[video]",
+hf = [
+    "datasets>=3.0.0",
     "huggingface-hub>=1.4.1",
     "hf>=1.7.1",
 ]
+egocentric = [
+    "macrodata-refiner[hf]",
+    "macrodata-refiner[video]",
+    "ego-vision[models]>=0.1.8",
+]
 text = [
     "warcio",
 ]
+hdf5 = [
+    "h5py",
+]
+zarr = [
+    "zarr>=2.18,<3",
+    "numcodecs<0.16",
+]
+mcap = [
+    "av",
+    "mcap",
+    "mcap-protobuf-support",
+    "mcap-ros2-support",
+    "pillow",
+]
 s3 = [
     "s3fs",
 ]
+tensorflow = [
+    "tensorflow",
+]
+tfds = [
+    "macrodata-refiner[tensorflow]",
+    "tensorflow-datasets",
+]
 testing = [
-    "macrodata-refiner[robotics]",
+    "macrodata-refiner[hdf5]",
+    "macrodata-refiner[hf]",
+    "macrodata-refiner[mcap]",
+    "macrodata-refiner[video]",
+    "macrodata-refiner[zarr]",
     "macrodata-refiner[text]",
     "macrodata-refiner[s3]",
+    "macrodata-refiner[tfds]",
     "pytest>=8.0.0",
     "pytest-cov>=5.0.0",
 ]
@@ -57,6 +88,7 @@ all = [
 
 [project.scripts]
 macrodata = "refiner.cli.main:main"
+mdr = "refiner.cli.main:main"
 
 [build-system]
 requires = ["setuptools>=77", "wheel"]

{macrodata_refiner-0.2.2 → macrodata_refiner-0.3.0}/src/macrodata_refiner.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.2.2
+Version: 0.3.0
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -13,30 +13,55 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: cloudpickle==3.1.2
-Requires-Dist: fsspec
+Requires-Dist: fsspec[http]
 Requires-Dist: httpx
 Requires-Dist: loguru
-Requires-Dist: opentelemetry-exporter-otlp-proto-http
-Requires-Dist: opentelemetry-sdk
 Requires-Dist: numpy
-Requires-Dist: psutil
 Requires-Dist: orjson
+Requires-Dist: packaging
 Requires-Dist: pyarrow
 Requires-Dist: msgspec>=0.20.0
+Requires-Dist: pydantic>=2.0.0
 Provides-Extra: video
 Requires-Dist: av; extra == "video"
-Provides-Extra: robotics
-Requires-Dist: macrodata-refiner[video]; extra == "robotics"
-Requires-Dist: huggingface-hub>=1.4.1; extra == "robotics"
-Requires-Dist: hf>=1.7.1; extra == "robotics"
+Requires-Dist: pillow; extra == "video"
+Provides-Extra: hf
+Requires-Dist: datasets>=3.0.0; extra == "hf"
+Requires-Dist: huggingface-hub>=1.4.1; extra == "hf"
+Requires-Dist: hf>=1.7.1; extra == "hf"
+Provides-Extra: egocentric
+Requires-Dist: macrodata-refiner[hf]; extra == "egocentric"
+Requires-Dist: macrodata-refiner[video]; extra == "egocentric"
+Requires-Dist: ego-vision[models]>=0.1.8; extra == "egocentric"
 Provides-Extra: text
 Requires-Dist: warcio; extra == "text"
+Provides-Extra: hdf5
+Requires-Dist: h5py; extra == "hdf5"
+Provides-Extra: zarr
+Requires-Dist: zarr<3,>=2.18; extra == "zarr"
+Requires-Dist: numcodecs<0.16; extra == "zarr"
+Provides-Extra: mcap
+Requires-Dist: av; extra == "mcap"
+Requires-Dist: mcap; extra == "mcap"
+Requires-Dist: mcap-protobuf-support; extra == "mcap"
+Requires-Dist: mcap-ros2-support; extra == "mcap"
+Requires-Dist: pillow; extra == "mcap"
 Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
+Provides-Extra: tensorflow
+Requires-Dist: tensorflow; extra == "tensorflow"
+Provides-Extra: tfds
+Requires-Dist: macrodata-refiner[tensorflow]; extra == "tfds"
+Requires-Dist: tensorflow-datasets; extra == "tfds"
 Provides-Extra: testing
-Requires-Dist: macrodata-refiner[robotics]; extra == "testing"
+Requires-Dist: macrodata-refiner[hdf5]; extra == "testing"
+Requires-Dist: macrodata-refiner[hf]; extra == "testing"
+Requires-Dist: macrodata-refiner[mcap]; extra == "testing"
+Requires-Dist: macrodata-refiner[video]; extra == "testing"
+Requires-Dist: macrodata-refiner[zarr]; extra == "testing"
 Requires-Dist: macrodata-refiner[text]; extra == "testing"
 Requires-Dist: macrodata-refiner[s3]; extra == "testing"
+Requires-Dist: macrodata-refiner[tfds]; extra == "testing"
 Requires-Dist: pytest>=8.0.0; extra == "testing"
 Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
 Provides-Extra: all
@@ -49,9 +74,10 @@ Dynamic: license-file
 
 <h1 align="center">Macrodata Refiner</h1>
 
-Refiner is an open-source engine for turning raw, unstructured, and multimodal data into **high-quality datasets** for large model training.
+Refiner is an open-source engine for turning raw robotics and multimodal data into **high-quality datasets** for model training.
 
-It replaces the brittle scripts and stitched-together data tooling that teams still use for training data work, while offering much better support for multimodal data, robotics workflows, and model-based processing.
+It gives training-data teams one pipeline model for multimodal data, robotics
+workflows, and model-based processing.
 
 It also plugs into the Macrodata platform, which gives you visibility into what is happening to your data while pipelines run: job and shard lifecycle, logs, metrics, manifests, and pipeline behavior. The same code can run locally for development and then scale out through Macrodata's elastic serverless cloud.
 
@@ -90,7 +116,7 @@ import refiner as mdr
             pad_frames=5,
         )
     )
-    .write_lerobot("hf://buckets/macrodata/test_bucket/aloha_motion")
+    .write_lerobot("hf://buckets/acme-robotics/aloha_motion")
     .launch_cloud(
         name="motion_trim",
         num_workers=4,
@@ -98,7 +124,7 @@ import refiner as mdr
 )
 ```
 
-Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+Need cloud GPUs? See [Resources, GPUs, and Services](docs/running-pipelines/resources-gpus-and-services.md).
 
 ### Local example
 
@@ -137,31 +163,33 @@ def add_preview(row):
 
 - training-data-first pipeline primitives instead of generic ETL abstractions
 - multimodal processing, with robotics support today
-- a lot of built-in readers, transforms, sinks, and lifecycle/runtime machinery so you do not have to rebuild the same scaffolding in scripts
+- built-in readers, transforms, sinks, and runtime machinery for common dataset work
 - access to any storage backend supported by `fsspec` (S3, GCP, Hugging Face, etc.)
 - local execution for development and elastic cloud execution for large runs
-- built-in observability through the Macrodata platform, so you can inspect how your data is changing instead of debugging blindly after the fact
+- built-in observability through the Macrodata platform for job state, logs, metrics, and manifests
 
 ## Docs
 
-Getting started:
+Start here:
 
-- [Pipeline basics](docs/pipeline-basics.md)
-- [Launchers](docs/launchers.md)
-- [CLI](docs/cli.md)
+- [Docs index](docs/index.md)
+- [Quickstart](docs/quickstart.md)
+- [Running pipelines](docs/running-pipelines/index.md)
 
-Core concepts:
+Build a dataset:
 
-- [Reading and writing data](docs/reading-and-writing.md)
-- [Transforms](docs/transforms.md)
-- [Expressions](docs/expressions.md)
-- [In-process debugging](docs/in-process-debugging.md)
-- [Task pipelines](docs/task-pipelines.md)
+- [Reading data](docs/reading-data/index.md)
+- [Episode data](docs/episode-data/index.md)
+- [Transforms](docs/transforms/index.md)
+- [Episode operations](docs/episode-operations/index.md)
+- [Writing data](docs/writing-data/index.md)
+- [Examples](docs/examples/index.md)
 
-Modalities and platform:
+Operate jobs:
 
-- [Robotics](docs/robotics.md)
-- [Observability](docs/observability.md)
+- [Platform](docs/platform/index.md)
+- [CLI](docs/cli/index.md)
+- [Reference](docs/reference/index.md)
 
 ## Community
 

{macrodata_refiner-0.2.2 → macrodata_refiner-0.3.0}/src/macrodata_refiner.egg-info/SOURCES.txt

@@ -8,11 +8,37 @@ src/macrodata_refiner.egg-info/entry_points.txt
 src/macrodata_refiner.egg-info/requires.txt
 src/macrodata_refiner.egg-info/top_level.txt
 src/refiner/__init__.py
+src/refiner/job_urls.py
 src/refiner/py.typed
 src/refiner/cli/__init__.py
 src/refiner/cli/auth.py
+src/refiner/cli/common.py
 src/refiner/cli/main.py
-src/refiner/cli/ui.py
+src/refiner/cli/secrets.py
+src/refiner/cli/commands/__init__.py
+src/refiner/cli/commands/auth.py
+src/refiner/cli/commands/jobs.py
+src/refiner/cli/commands/run.py
+src/refiner/cli/commands/secrets.py
+src/refiner/cli/jobs/__init__.py
+src/refiner/cli/jobs/attach.py
+src/refiner/cli/jobs/common.py
+src/refiner/cli/jobs/control.py
+src/refiner/cli/jobs/follow.py
+src/refiner/cli/jobs/get.py
+src/refiner/cli/jobs/list.py
+src/refiner/cli/jobs/logs.py
+src/refiner/cli/jobs/manifest.py
+src/refiner/cli/jobs/metrics.py
+src/refiner/cli/jobs/workers.py
+src/refiner/cli/run/__init__.py
+src/refiner/cli/run/cloud.py
+src/refiner/cli/run/command.py
+src/refiner/cli/run/local.py
+src/refiner/cli/run/modes.py
+src/refiner/cli/ui/__init__.py
+src/refiner/cli/ui/console.py
+src/refiner/cli/ui/terminal.py
 src/refiner/execution/__init__.py
 src/refiner/execution/buffer.py
 src/refiner/execution/engine.py
@@ -24,6 +50,25 @@ src/refiner/execution/operators/row.py
 src/refiner/execution/operators/vectorized.py
 src/refiner/execution/tracking/__init__.py
 src/refiner/execution/tracking/shards.py
+src/refiner/inference/__init__.py
+src/refiner/inference/capabilities.py
+src/refiner/inference/generate_pooling.py
+src/refiner/inference/generate_text.py
+src/refiner/inference/types.py
+src/refiner/inference/internal/__init__.py
+src/refiner/inference/internal/media.py
+src/refiner/inference/internal/message_conversion.py
+src/refiner/inference/internal/response.py
+src/refiner/inference/internal/runtime.py
+src/refiner/inference/internal/schema.py
+src/refiner/inference/internal/transport.py
+src/refiner/inference/internal/usage.py
+src/refiner/inference/providers/__init__.py
+src/refiner/inference/providers/anthropic.py
+src/refiner/inference/providers/base.py
+src/refiner/inference/providers/google.py
+src/refiner/inference/providers/openai.py
+src/refiner/inference/providers/warnings.py
 src/refiner/io/__init__.py
 src/refiner/io/datafile.py
 src/refiner/io/datafolder.py
@@ -32,21 +77,29 @@ src/refiner/launchers/__init__.py
 src/refiner/launchers/base.py
 src/refiner/launchers/cloud.py
 src/refiner/launchers/local.py
+src/refiner/launchers/secrets.py
 src/refiner/pipeline/__init__.py
 src/refiner/pipeline/expressions.py
 src/refiner/pipeline/pipeline.py
 src/refiner/pipeline/planning.py
+src/refiner/pipeline/resources.py
 src/refiner/pipeline/steps.py
 src/refiner/pipeline/data/block.py
+src/refiner/pipeline/data/datatype.py
 src/refiner/pipeline/data/row.py
 src/refiner/pipeline/data/shard.py
 src/refiner/pipeline/data/tabular.py
 src/refiner/pipeline/sinks/__init__.py
+src/refiner/pipeline/sinks/assets.py
 src/refiner/pipeline/sinks/base.py
 src/refiner/pipeline/sinks/jsonl.py
 src/refiner/pipeline/sinks/lerobot.py
-src/refiner/pipeline/sinks/lerobot_reducer.py
 src/refiner/pipeline/sinks/parquet.py
+src/refiner/pipeline/sinks/zarr.py
+src/refiner/pipeline/sinks/reducer/__init__.py
+src/refiner/pipeline/sinks/reducer/file.py
+src/refiner/pipeline/sinks/reducer/lerobot.py
+src/refiner/pipeline/sinks/reducer/zarr.py
 src/refiner/pipeline/sources/__init__.py
 src/refiner/pipeline/sources/base.py
 src/refiner/pipeline/sources/items.py
@@ -54,10 +107,17 @@ src/refiner/pipeline/sources/task.py
 src/refiner/pipeline/sources/readers/__init__.py
 src/refiner/pipeline/sources/readers/base.py
 src/refiner/pipeline/sources/readers/csv.py
-src/refiner/pipeline/sources/readers/jsonl.py
+src/refiner/pipeline/sources/readers/files.py
+src/refiner/pipeline/sources/readers/hdf5.py
+src/refiner/pipeline/sources/readers/hf_dataset.py
+src/refiner/pipeline/sources/readers/json.py
 src/refiner/pipeline/sources/readers/lerobot.py
+src/refiner/pipeline/sources/readers/mcap.py
 src/refiner/pipeline/sources/readers/parquet.py
+src/refiner/pipeline/sources/readers/tfds.py
+src/refiner/pipeline/sources/readers/tfrecord.py
 src/refiner/pipeline/sources/readers/utils.py
+src/refiner/pipeline/sources/readers/zarr.py
 src/refiner/pipeline/utils/__init__.py
 src/refiner/pipeline/utils/cache/__init__.py
 src/refiner/pipeline/utils/cache/decoder_cache.py
@@ -68,11 +128,16 @@ src/refiner/platform/auth.py
 src/refiner/platform/manifest.py
 src/refiner/platform/client/__init__.py
 src/refiner/platform/client/api.py
-src/refiner/platform/client/http.py
 src/refiner/platform/client/models.py
 src/refiner/platform/client/serialize.py
 src/refiner/robotics/__init__.py
+src/refiner/robotics/egocentric.py
 src/refiner/robotics/motion.py
+src/refiner/robotics/reward.py
+src/refiner/robotics/row.py
+src/refiner/robotics/subtask_annotation.py
+src/refiner/robotics/synchronization.py
+src/refiner/robotics/tabular.py
 src/refiner/robotics/lerobot_format/__init__.py
 src/refiner/robotics/lerobot_format/row.py
 src/refiner/robotics/lerobot_format/tabular.py
@@ -81,11 +146,17 @@ src/refiner/robotics/lerobot_format/metadata/info.py
 src/refiner/robotics/lerobot_format/metadata/metadata.py
 src/refiner/robotics/lerobot_format/metadata/stats.py
 src/refiner/robotics/lerobot_format/metadata/tasks.py
+src/refiner/services/__init__.py
+src/refiner/services/base.py
+src/refiner/services/discovery.py
+src/refiner/services/manager.py
+src/refiner/services/vllm.py
 src/refiner/text/__init__.py
 src/refiner/text/commoncrawl.py
 src/refiner/utils/__init__.py
 src/refiner/utils/imports.py
 src/refiner/video/__init__.py
+src/refiner/video/decode.py
 src/refiner/video/remux.py
 src/refiner/video/transcode.py
 src/refiner/video/types.py
@@ -93,25 +164,17 @@ src/refiner/video/writer.py
 src/refiner/worker/__init__.py
 src/refiner/worker/context.py
 src/refiner/worker/entrypoint.py
+src/refiner/worker/lifecycle.py
 src/refiner/worker/runner.py
 src/refiner/worker/workdir.py
-src/refiner/worker/lifecycle/__init__.py
-src/refiner/worker/lifecycle/base.py
-src/refiner/worker/lifecycle/platform.py
-src/refiner/worker/lifecycle/local/__init__.py
-src/refiner/worker/lifecycle/local/claim.py
-src/refiner/worker/lifecycle/local/files.py
-src/refiner/worker/lifecycle/local/lifecycle.py
 src/refiner/worker/metrics/__init__.py
 src/refiner/worker/metrics/api.py
-src/refiner/worker/metrics/context.py
-src/refiner/worker/metrics/otel.py
+src/refiner/worker/metrics/emitter.py
 src/refiner/worker/resources/__init__.py
 src/refiner/worker/resources/cpu.py
 src/refiner/worker/resources/gpu.py
-src/refiner/worker/resources/memory.py
-src/refiner/worker/resources/network.py
 tests/test_cache.py
 tests/test_commoncrawl_text.py
 tests/test_expressions.py
-tests/test_optional_dependencies.py
+tests/test_optional_dependencies.py
+tests/test_video_decode.py

{macrodata_refiner-0.2.2 → macrodata_refiner-0.3.0}/src/macrodata_refiner.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
 macrodata = refiner.cli.main:main
+mdr = refiner.cli.main:main