macrodata-refiner 0.2.0__tar.gz → 0.2.2__tar.gz

{macrodata_refiner-0.2.0/src/macrodata_refiner.egg-info → macrodata_refiner-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.2.0
+Version: 0.2.2
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -12,12 +12,10 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: av
 Requires-Dist: cloudpickle==3.1.2
 Requires-Dist: fsspec
 Requires-Dist: httpx
 Requires-Dist: loguru
-Requires-Dist: huggingface-hub>=1.4.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: numpy
@@ -25,7 +23,24 @@ Requires-Dist: psutil
 Requires-Dist: orjson
 Requires-Dist: pyarrow
 Requires-Dist: msgspec>=0.20.0
-Requires-Dist: hf>=1.7.1
+Provides-Extra: video
+Requires-Dist: av; extra == "video"
+Provides-Extra: robotics
+Requires-Dist: macrodata-refiner[video]; extra == "robotics"
+Requires-Dist: huggingface-hub>=1.4.1; extra == "robotics"
+Requires-Dist: hf>=1.7.1; extra == "robotics"
+Provides-Extra: text
+Requires-Dist: warcio; extra == "text"
+Provides-Extra: s3
+Requires-Dist: s3fs; extra == "s3"
+Provides-Extra: testing
+Requires-Dist: macrodata-refiner[robotics]; extra == "testing"
+Requires-Dist: macrodata-refiner[text]; extra == "testing"
+Requires-Dist: macrodata-refiner[s3]; extra == "testing"
+Requires-Dist: pytest>=8.0.0; extra == "testing"
+Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
+Provides-Extra: all
+Requires-Dist: macrodata-refiner[testing]; extra == "all"
 Dynamic: license-file
 
 <p align="center">
@@ -83,6 +98,8 @@ import refiner as mdr
 )
 ```
 
+Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+
 ### Local example
 
 Launch a local pipeline:

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/README.md

@@ -53,6 +53,8 @@ import refiner as mdr
 )
 ```
 
+Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+
 ### Local example
 
 Launch a local pipeline:

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "macrodata-refiner"
-version = "0.2.0"
+version = "0.2.2"
 description = "Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets"
 readme = "README.md"
 license = "Apache-2.0"
@@ -16,12 +16,10 @@ authors = [
 ]
 requires-python = ">=3.10"
 dependencies = [
-    "av",
     "cloudpickle==3.1.2",
     "fsspec",
     "httpx",
     "loguru",
-    "huggingface-hub>=1.4.1",
     "opentelemetry-exporter-otlp-proto-http",
     "opentelemetry-sdk",
     "numpy",
@@ -29,8 +27,33 @@ dependencies = [
     "orjson",
     "pyarrow",
     "msgspec>=0.20.0",
+]
+
+[project.optional-dependencies]
+video = [
+    "av",
+]
+robotics = [
+    "macrodata-refiner[video]",
+    "huggingface-hub>=1.4.1",
     "hf>=1.7.1",
 ]
+text = [
+    "warcio",
+]
+s3 = [
+    "s3fs",
+]
+testing = [
+    "macrodata-refiner[robotics]",
+    "macrodata-refiner[text]",
+    "macrodata-refiner[s3]",
+    "pytest>=8.0.0",
+    "pytest-cov>=5.0.0",
+]
+all = [
+    "macrodata-refiner[testing]",
+]
 
 [project.scripts]
 macrodata = "refiner.cli.main:main"
@@ -47,9 +70,8 @@ refiner = ["py.typed"]
 
 [dependency-groups]
 dev = [
+    "macrodata-refiner[all]",
     "pre-commit>=4.0.0",
-    "pytest>=8.0.0",
-    "pytest-cov>=5.0.0",
     "ruff>=0.14.10",
     "ty>=0.0.7",
 ]

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2/src/macrodata_refiner.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.2.0
+Version: 0.2.2
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -12,12 +12,10 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: av
 Requires-Dist: cloudpickle==3.1.2
 Requires-Dist: fsspec
 Requires-Dist: httpx
 Requires-Dist: loguru
-Requires-Dist: huggingface-hub>=1.4.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: numpy
@@ -25,7 +23,24 @@ Requires-Dist: psutil
 Requires-Dist: orjson
 Requires-Dist: pyarrow
 Requires-Dist: msgspec>=0.20.0
-Requires-Dist: hf>=1.7.1
+Provides-Extra: video
+Requires-Dist: av; extra == "video"
+Provides-Extra: robotics
+Requires-Dist: macrodata-refiner[video]; extra == "robotics"
+Requires-Dist: huggingface-hub>=1.4.1; extra == "robotics"
+Requires-Dist: hf>=1.7.1; extra == "robotics"
+Provides-Extra: text
+Requires-Dist: warcio; extra == "text"
+Provides-Extra: s3
+Requires-Dist: s3fs; extra == "s3"
+Provides-Extra: testing
+Requires-Dist: macrodata-refiner[robotics]; extra == "testing"
+Requires-Dist: macrodata-refiner[text]; extra == "testing"
+Requires-Dist: macrodata-refiner[s3]; extra == "testing"
+Requires-Dist: pytest>=8.0.0; extra == "testing"
+Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
+Provides-Extra: all
+Requires-Dist: macrodata-refiner[testing]; extra == "all"
 Dynamic: license-file
 
 <p align="center">
@@ -83,6 +98,8 @@ import refiner as mdr
 )
 ```
 
+Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+
 ### Local example
 
 Launch a local pipeline:

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/src/macrodata_refiner.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ src/refiner/cli/ui.py
 src/refiner/execution/__init__.py
 src/refiner/execution/buffer.py
 src/refiner/execution/engine.py
+src/refiner/execution/asyncio/__init__.py
 src/refiner/execution/asyncio/runtime.py
 src/refiner/execution/asyncio/window.py
 src/refiner/execution/operators/__init__.py
@@ -31,12 +32,6 @@ src/refiner/launchers/__init__.py
 src/refiner/launchers/base.py
 src/refiner/launchers/cloud.py
 src/refiner/launchers/local.py
-src/refiner/media/__init__.py
-src/refiner/media/video/__init__.py
-src/refiner/media/video/remux.py
-src/refiner/media/video/transcode.py
-src/refiner/media/video/types.py
-src/refiner/media/video/writer.py
 src/refiner/pipeline/__init__.py
 src/refiner/pipeline/expressions.py
 src/refiner/pipeline/pipeline.py
@@ -86,6 +81,15 @@ src/refiner/robotics/lerobot_format/metadata/info.py
 src/refiner/robotics/lerobot_format/metadata/metadata.py
 src/refiner/robotics/lerobot_format/metadata/stats.py
 src/refiner/robotics/lerobot_format/metadata/tasks.py
+src/refiner/text/__init__.py
+src/refiner/text/commoncrawl.py
+src/refiner/utils/__init__.py
+src/refiner/utils/imports.py
+src/refiner/video/__init__.py
+src/refiner/video/remux.py
+src/refiner/video/transcode.py
+src/refiner/video/types.py
+src/refiner/video/writer.py
 src/refiner/worker/__init__.py
 src/refiner/worker/context.py
 src/refiner/worker/entrypoint.py
@@ -104,7 +108,10 @@ src/refiner/worker/metrics/context.py
 src/refiner/worker/metrics/otel.py
 src/refiner/worker/resources/__init__.py
 src/refiner/worker/resources/cpu.py
+src/refiner/worker/resources/gpu.py
 src/refiner/worker/resources/memory.py
 src/refiner/worker/resources/network.py
 tests/test_cache.py
-tests/test_expressions.py
+tests/test_commoncrawl_text.py
+tests/test_expressions.py
+tests/test_optional_dependencies.py

macrodata_refiner-0.2.2/src/macrodata_refiner.egg-info/requires.txt

@@ -0,0 +1,35 @@
+cloudpickle==3.1.2
+fsspec
+httpx
+loguru
+opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk
+numpy
+psutil
+orjson
+pyarrow
+msgspec>=0.20.0
+
+[all]
+macrodata-refiner[testing]
+
+[robotics]
+macrodata-refiner[video]
+huggingface-hub>=1.4.1
+hf>=1.7.1
+
+[s3]
+s3fs
+
+[testing]
+macrodata-refiner[robotics]
+macrodata-refiner[text]
+macrodata-refiner[s3]
+pytest>=8.0.0
+pytest-cov>=5.0.0
+
+[text]
+warcio
+
+[video]
+av

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/src/refiner/__init__.py

@@ -1,11 +1,9 @@
+import refiner.io as io
+import refiner.pipeline as pipeline
 import refiner.robotics as robotics
-from refiner.io import DataFile, DataFileSet, DataFolder
-from refiner.launchers import LaunchStats, LocalLauncher
-from refiner.media import VideoFile
+import refiner.text as text
+import refiner.video as video
 from refiner.pipeline import (
-    RefinerPipeline,
-    Row,
-    Shard,
     from_items,
     from_source,
     read_csv,
@@ -22,19 +20,11 @@ from refiner.worker.metrics.api import (
     log_throughput,
     register_gauge,
 )
-from refiner.worker.runner import Worker, WorkerRunStats
+
+robot = robotics
 
 __all__ = [
-    "RefinerPipeline",
-    "LocalLauncher",
-    "LaunchStats",
-    "DataFile",
-    "DataFolder",
-    "DataFileSet",
-    "Shard",
-    "Row",
-    "Worker",
-    "WorkerRunStats",
+    # sources
     "read_csv",
     "read_jsonl",
     "read_lerobot",
@@ -42,16 +32,22 @@ __all__ = [
     "from_items",
     "from_source",
     "task",
+    # metrics
     "log_throughput",
     "log_gauge",
     "log_gauges",
-    "register_gauge",
     "log_histogram",
+    "register_gauge",
+    # expressions
     "col",
     "lit",
     "coalesce",
     "if_else",
-    "VideoFile",
-    "Video",
+    # submodules
+    "io",
+    "pipeline",
+    "video",
+    "robot",
     "robotics",
+    "text",
 ]

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/src/refiner/cli/auth.py

@@ -18,7 +18,7 @@ from refiner.platform.client import (
     sanitize_terminal_text,
     verify_api_key,
 )
-from refiner.cli.ui import display_identity, print_banner
+from refiner.cli.ui import display_identity, print_banner, stdin_is_interactive
 
 _TOKEN_SETTINGS_SUFFIX = "/settings/api-keys"
 
@@ -31,7 +31,7 @@ def _read_token(args: argparse.Namespace) -> str:
     if args.token and args.token.strip():
         return args.token.strip()
 
-    read_from_stdin = args.token_stdin or not sys.stdin.isatty()
+    read_from_stdin = args.token_stdin or not stdin_is_interactive()
     if read_from_stdin:
         token = sys.stdin.read().strip()
         if token:

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/src/refiner/cli/ui.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import sys
+
 from refiner.platform.client import UserIdentity
 
 ASCII_BANNER = r"""
@@ -26,3 +28,10 @@ def display_identity(user: UserIdentity) -> str:
     if email:
         return f"{label} ({email})"
     return label
+
+
+def stdin_is_interactive() -> bool:
+    try:
+        return sys.stdin.isatty()
+    except Exception:  # pragma: no cover
+        return False

macrodata_refiner-0.2.2/src/refiner/execution/asyncio/__init__.py

@@ -0,0 +1 @@
+

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/src/refiner/execution/operators/row.py

@@ -58,15 +58,16 @@ def execute_row_steps(
             )
 
     async def _run_async_step(*, step: AsyncRowStep, row: Row) -> Row:
-        result = step.apply_row_async(row)
-        if inspect.isawaitable(result):
-            result = await result
-        result = cast(MapResult, result)
-        if isinstance(result, Row):
-            return result
-        if isinstance(result, dict):
-            return row.update(result)
-        raise TypeError(f"Unsupported map_async() result type: {type(result)!r}")
+        with set_active_step_index(step.index):
+            result = step.apply_row_async(row)
+            if inspect.isawaitable(result):
+                result = await result
+            result = cast(MapResult, result)
+            if isinstance(result, Row):
+                return result
+            if isinstance(result, dict):
+                return row.update(result)
+            raise TypeError(f"Unsupported map_async() result type: {type(result)!r}")
 
     def _run_step(i: int, *, flush_all: bool) -> None:
         step = ordered[i]

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/src/refiner/execution/operators/vectorized.py

@@ -10,7 +10,7 @@ from refiner.execution.tracking.shards import (
     count_table_by_shard,
     counts_delta,
 )
-from refiner.pipeline.data.tabular import repeat_scalar
+from refiner.pipeline.data.tabular import filter_table, repeat_scalar
 from refiner.pipeline.expressions import eval_expr_arrow
 from refiner.pipeline.steps import (
     CastStep,
@@ -22,6 +22,7 @@ from refiner.pipeline.steps import (
     VectorizedOp,
     WithColumnsStep,
 )
+from refiner.worker.context import set_active_step_index
 from refiner.worker.metrics.api import log_throughput
 
 
@@ -68,14 +69,7 @@ def apply_vectorized_op(
         return out, None
 
     if isinstance(op, FilterExprStep):
-        mask = eval_expr_arrow(op.predicate, table)
-        next_table = (
-            table
-            if isinstance(mask, pa.Scalar) and bool(mask.as_py())
-            else (
-                table.slice(0, 0) if isinstance(mask, pa.Scalar) else table.filter(mask)
-            )
-        )
+        next_table = filter_table(table, op.predicate)
         next_shard_counts = count_table_by_shard(next_table)
         for shard_id in set(shard_counts) | set(next_shard_counts):
             previous = int(shard_counts.get(shard_id, 0))
@@ -100,7 +94,8 @@ def apply_vectorized_op(
         return next_table, next_shard_counts
 
     if isinstance(op, FnTableStep):
-        next_table = op.fn(table)
+        with set_active_step_index(op.index):
+            next_table = op.fn(table)
         if not isinstance(next_table, pa.Table):
             raise TypeError(
                 f"map_table() must return pa.Table, got {type(next_table)!r}"

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/src/refiner/io/datafolder.py

@@ -1,4 +1,4 @@
-from collections.abc import Iterable, Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from os import PathLike
 from typing import IO, Any, TypeAlias, Union, cast
 
@@ -113,6 +113,34 @@ class DataFolder(DirFileSystem):
             return self.abs_path(paths)
         return [self.abs_path(p) for p in paths]
 
+    def find(self, path: str, *args, **kwargs):
+        # Avoid DirFileSystem.find(): some backends (notably HF buckets) can leak
+        # sibling prefix matches like `root-2/...` or return the bare root entry,
+        # and DirFileSystem._relpath() asserts before we can filter them out.
+        """List paths under this folder, skipping backend results outside the base path."""
+        detail = kwargs.get("detail", False)
+        target = self._join(path.rstrip("/"))
+        ret = self.fs.find(target, *args, **kwargs)
+        target = target.rstrip("/")
+        target_prefix = target + self.fs.sep
+        alt_target = target[1:] if target.startswith(self.fs.sep) else None
+        alt_prefix = alt_target + self.fs.sep if alt_target is not None else None
+
+        def rel(p: str) -> str | None:
+            if p == target or (alt_target is not None and p == alt_target):
+                return path.rstrip("/")
+            if p.startswith(target_prefix):
+                suffix = p[len(target_prefix) :]
+            elif alt_prefix is not None and p.startswith(alt_prefix):
+                suffix = p[len(alt_prefix) :]
+            else:
+                return None
+            return suffix if path in {"", "/"} else f"{path.rstrip('/')}/{suffix}"
+
+        if detail:
+            return {r: info for p, info in ret.items() if (r := rel(p)) is not None}
+        return [r for p in ret if (r := rel(p)) is not None]
+
     def open_files(
         self, paths: Iterable[str], mode: str = "rb", **kwargs
     ) -> list[IO[Any]]:
@@ -159,3 +187,29 @@ class DataFolder(DirFileSystem):
 
     def files(self, relpaths: Iterable[str]) -> list[DataFile]:
         return [self.file(p) for p in relpaths]
+
+    def iter_files_with_sizes(
+        self, *, recursive: bool = False, **kwargs: Any
+    ) -> Iterator[tuple[DataFile, int | None]]:
+        if recursive:
+            found = self.find("", detail=True, **kwargs)
+            items: Iterable[tuple[str, Mapping[str, Any]]] = found.items()
+        else:
+            items = (
+                (str(info["name"]), info)
+                for info in self.ls("", detail=True, **kwargs)
+                if isinstance(info, Mapping)
+            )
+
+        for relpath, info in sorted(items, key=lambda item: item[0]):
+            info_dict = dict(info)
+            if info_dict.get("type") != "file":
+                continue
+            size = info_dict.get("size")
+            yield self.file(relpath), size if isinstance(size, int) else None
+
+    def iter_files(
+        self, *, recursive: bool = False, **kwargs: Any
+    ) -> Iterator[DataFile]:
+        for file, _ in self.iter_files_with_sizes(recursive=recursive, **kwargs):
+            yield file

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.2}/src/refiner/io/fileset.py

@@ -170,62 +170,77 @@ class DataFileSet:
         exts = tuple(e.lower() for e in self.extensions)
         seen: set[tuple[int, str]] = set()
         expanded: list[tuple[DataFile, ...]] = []
-
-        def _append_file(out: list[DataFile], file: DataFile) -> None:
-            if exts and not file.path.lower().endswith(exts):
+        sizes = dict(self._sizes)
+
+        def _append_file(
+            out: list[DataFile],
+            file: DataFile,
+            *,
+            size: int | None = None,
+            apply_extensions: bool = True,
+        ) -> None:
+            if apply_extensions and exts and not file.path.lower().endswith(exts):
                 return
             key = (id(file.fs), file.path)
             if key in seen:
                 return
             seen.add(key)
             out.append(file)
+            if size is not None:
+                sizes[(len(expanded), file.abs_path())] = int(size)
 
         for entry in self.entries:
             files: list[DataFile] = []
+            if isinstance(entry, _PathSource) and not glob.has_magic(entry.path):
+                try:
+                    info = entry.fs.info(entry.path)
+                except FileNotFoundError:
+                    raise FileNotFoundError(
+                        f"Could not resolve input: {entry.fs.unstrip_protocol(entry.path)!r}"
+                    )
+                item_type = info.get("type")
+                if item_type == "directory":
+                    entry = DataFolder(path=entry.path, fs=entry.fs)
+                elif item_type == "file":
+                    entry = DataFile(fs=entry.fs, path=entry.path)
+                else:
+                    raise TypeError(
+                        f"Unsupported file type {item_type!r} for input: "
+                        f"{entry.fs.unstrip_protocol(entry.path)!r}"
+                    )
+
             if isinstance(entry, DataFile):
-                _append_file(files, entry)
+                _append_file(files, entry, apply_extensions=False)
             elif isinstance(entry, DataFolder):
-                paths = (
-                    sorted(entry.find(""))
-                    if self.recursive
-                    else sorted(
-                        e["name"] if isinstance(e, dict) else e
-                        for e in entry.ls("", detail=True)
-                        if not isinstance(e, dict) or e.get("type") == "file"
-                    )
-                )
-                for path in paths:
-                    _append_file(files, entry.file(path))
+                for file, size in entry.iter_files_with_sizes(recursive=self.recursive):
+                    _append_file(files, file, size=size)
             else:
                 next_fs, path = entry.fs, entry.path
                 if glob.has_magic(path):
-                    for expanded_path in sorted(next_fs.glob(path)):
-                        _append_file(files, DataFile(fs=next_fs, path=expanded_path))
-                elif next_fs.exists(path):
-                    if next_fs.isdir(path):
-                        paths = (
-                            sorted(next_fs.find(path))
-                            if self.recursive
-                            else sorted(
-                                e["name"] if isinstance(e, dict) else e
-                                for e in next_fs.ls(path, detail=True)
-                                if not isinstance(e, dict) or e.get("type") == "file"
-                            )
+                    matched = next_fs.glob(path, detail=True)
+                    items = matched.items()
+                    for expanded_path, info in sorted(items):
+                        if not isinstance(expanded_path, str) or not isinstance(
+                            info, Mapping
+                        ):
+                            continue
+                        if info.get("type") != "file":
+                            continue
+                        size = info.get("size")
+                        _append_file(
+                            files,
+                            DataFile(fs=next_fs, path=expanded_path),
+                            size=size if isinstance(size, int) else None,
                         )
-                        for expanded_path in paths:
-                            _append_file(
-                                files, DataFile(fs=next_fs, path=expanded_path)
-                            )
-                    else:
-                        _append_file(files, DataFile(fs=next_fs, path=path))
                 else:
-                    raise FileNotFoundError(
-                        f"Could not resolve input: {next_fs.unstrip_protocol(path)!r}"
+                    raise AssertionError(
+                        "non-glob _PathSource should have been resolved"
                     )
             expanded.append(tuple(files))
 
         out = tuple(expanded)
         object.__setattr__(self, "_expanded_sources", out)
+        object.__setattr__(self, "_sizes", sizes)
         return out
 
     def resolve_file(self, source_index: int, path: str) -> DataFile: