macrodata-refiner 0.2.1__tar.gz → 0.2.2__tar.gz

{macrodata_refiner-0.2.1/src/macrodata_refiner.egg-info → macrodata_refiner-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.2.1
+Version: 0.2.2
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -12,12 +12,10 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: av
 Requires-Dist: cloudpickle==3.1.2
 Requires-Dist: fsspec
 Requires-Dist: httpx
 Requires-Dist: loguru
-Requires-Dist: huggingface-hub>=1.4.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: numpy
@@ -25,7 +23,24 @@ Requires-Dist: psutil
 Requires-Dist: orjson
 Requires-Dist: pyarrow
 Requires-Dist: msgspec>=0.20.0
-Requires-Dist: hf>=1.7.1
+Provides-Extra: video
+Requires-Dist: av; extra == "video"
+Provides-Extra: robotics
+Requires-Dist: macrodata-refiner[video]; extra == "robotics"
+Requires-Dist: huggingface-hub>=1.4.1; extra == "robotics"
+Requires-Dist: hf>=1.7.1; extra == "robotics"
+Provides-Extra: text
+Requires-Dist: warcio; extra == "text"
+Provides-Extra: s3
+Requires-Dist: s3fs; extra == "s3"
+Provides-Extra: testing
+Requires-Dist: macrodata-refiner[robotics]; extra == "testing"
+Requires-Dist: macrodata-refiner[text]; extra == "testing"
+Requires-Dist: macrodata-refiner[s3]; extra == "testing"
+Requires-Dist: pytest>=8.0.0; extra == "testing"
+Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
+Provides-Extra: all
+Requires-Dist: macrodata-refiner[testing]; extra == "all"
 Dynamic: license-file
 
 <p align="center">
@@ -83,6 +98,8 @@ import refiner as mdr
 )
 ```
 
+Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+
 ### Local example
 
 Launch a local pipeline:

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2}/README.md

@@ -53,6 +53,8 @@ import refiner as mdr
 )
 ```
 
+Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+
 ### Local example
 
 Launch a local pipeline:

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "macrodata-refiner"
-version = "0.2.1"
+version = "0.2.2"
 description = "Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets"
 readme = "README.md"
 license = "Apache-2.0"
@@ -16,12 +16,10 @@ authors = [
 ]
 requires-python = ">=3.10"
 dependencies = [
-    "av",
     "cloudpickle==3.1.2",
     "fsspec",
     "httpx",
     "loguru",
-    "huggingface-hub>=1.4.1",
     "opentelemetry-exporter-otlp-proto-http",
     "opentelemetry-sdk",
     "numpy",
@@ -29,8 +27,33 @@ dependencies = [
     "orjson",
     "pyarrow",
     "msgspec>=0.20.0",
+]
+
+[project.optional-dependencies]
+video = [
+    "av",
+]
+robotics = [
+    "macrodata-refiner[video]",
+    "huggingface-hub>=1.4.1",
     "hf>=1.7.1",
 ]
+text = [
+    "warcio",
+]
+s3 = [
+    "s3fs",
+]
+testing = [
+    "macrodata-refiner[robotics]",
+    "macrodata-refiner[text]",
+    "macrodata-refiner[s3]",
+    "pytest>=8.0.0",
+    "pytest-cov>=5.0.0",
+]
+all = [
+    "macrodata-refiner[testing]",
+]
 
 [project.scripts]
 macrodata = "refiner.cli.main:main"
@@ -47,9 +70,8 @@ refiner = ["py.typed"]
 
 [dependency-groups]
 dev = [
+    "macrodata-refiner[all]",
     "pre-commit>=4.0.0",
-    "pytest>=8.0.0",
-    "pytest-cov>=5.0.0",
     "ruff>=0.14.10",
     "ty>=0.0.7",
 ]

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2/src/macrodata_refiner.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.2.1
+Version: 0.2.2
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -12,12 +12,10 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: av
 Requires-Dist: cloudpickle==3.1.2
 Requires-Dist: fsspec
 Requires-Dist: httpx
 Requires-Dist: loguru
-Requires-Dist: huggingface-hub>=1.4.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: numpy
@@ -25,7 +23,24 @@ Requires-Dist: psutil
 Requires-Dist: orjson
 Requires-Dist: pyarrow
 Requires-Dist: msgspec>=0.20.0
-Requires-Dist: hf>=1.7.1
+Provides-Extra: video
+Requires-Dist: av; extra == "video"
+Provides-Extra: robotics
+Requires-Dist: macrodata-refiner[video]; extra == "robotics"
+Requires-Dist: huggingface-hub>=1.4.1; extra == "robotics"
+Requires-Dist: hf>=1.7.1; extra == "robotics"
+Provides-Extra: text
+Requires-Dist: warcio; extra == "text"
+Provides-Extra: s3
+Requires-Dist: s3fs; extra == "s3"
+Provides-Extra: testing
+Requires-Dist: macrodata-refiner[robotics]; extra == "testing"
+Requires-Dist: macrodata-refiner[text]; extra == "testing"
+Requires-Dist: macrodata-refiner[s3]; extra == "testing"
+Requires-Dist: pytest>=8.0.0; extra == "testing"
+Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
+Provides-Extra: all
+Requires-Dist: macrodata-refiner[testing]; extra == "all"
 Dynamic: license-file
 
 <p align="center">
@@ -83,6 +98,8 @@ import refiner as mdr
 )
 ```
 
+Need cloud GPUs? See [Launchers](docs/launchers.md) for the GPU-specific cloud options.
+
 ### Local example
 
 Launch a local pipeline:

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2}/src/macrodata_refiner.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ src/refiner/cli/ui.py
 src/refiner/execution/__init__.py
 src/refiner/execution/buffer.py
 src/refiner/execution/engine.py
+src/refiner/execution/asyncio/__init__.py
 src/refiner/execution/asyncio/runtime.py
 src/refiner/execution/asyncio/window.py
 src/refiner/execution/operators/__init__.py
@@ -31,12 +32,6 @@ src/refiner/launchers/__init__.py
 src/refiner/launchers/base.py
 src/refiner/launchers/cloud.py
 src/refiner/launchers/local.py
-src/refiner/media/__init__.py
-src/refiner/media/video/__init__.py
-src/refiner/media/video/remux.py
-src/refiner/media/video/transcode.py
-src/refiner/media/video/types.py
-src/refiner/media/video/writer.py
 src/refiner/pipeline/__init__.py
 src/refiner/pipeline/expressions.py
 src/refiner/pipeline/pipeline.py
@@ -86,6 +81,15 @@ src/refiner/robotics/lerobot_format/metadata/info.py
 src/refiner/robotics/lerobot_format/metadata/metadata.py
 src/refiner/robotics/lerobot_format/metadata/stats.py
 src/refiner/robotics/lerobot_format/metadata/tasks.py
+src/refiner/text/__init__.py
+src/refiner/text/commoncrawl.py
+src/refiner/utils/__init__.py
+src/refiner/utils/imports.py
+src/refiner/video/__init__.py
+src/refiner/video/remux.py
+src/refiner/video/transcode.py
+src/refiner/video/types.py
+src/refiner/video/writer.py
 src/refiner/worker/__init__.py
 src/refiner/worker/context.py
 src/refiner/worker/entrypoint.py
@@ -104,7 +108,10 @@ src/refiner/worker/metrics/context.py
 src/refiner/worker/metrics/otel.py
 src/refiner/worker/resources/__init__.py
 src/refiner/worker/resources/cpu.py
+src/refiner/worker/resources/gpu.py
 src/refiner/worker/resources/memory.py
 src/refiner/worker/resources/network.py
 tests/test_cache.py
-tests/test_expressions.py
+tests/test_commoncrawl_text.py
+tests/test_expressions.py
+tests/test_optional_dependencies.py

macrodata_refiner-0.2.2/src/macrodata_refiner.egg-info/requires.txt

@@ -0,0 +1,35 @@
+cloudpickle==3.1.2
+fsspec
+httpx
+loguru
+opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk
+numpy
+psutil
+orjson
+pyarrow
+msgspec>=0.20.0
+
+[all]
+macrodata-refiner[testing]
+
+[robotics]
+macrodata-refiner[video]
+huggingface-hub>=1.4.1
+hf>=1.7.1
+
+[s3]
+s3fs
+
+[testing]
+macrodata-refiner[robotics]
+macrodata-refiner[text]
+macrodata-refiner[s3]
+pytest>=8.0.0
+pytest-cov>=5.0.0
+
+[text]
+warcio
+
+[video]
+av

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2}/src/refiner/__init__.py

@@ -1,11 +1,9 @@
+import refiner.io as io
+import refiner.pipeline as pipeline
 import refiner.robotics as robotics
-from refiner.io import DataFile, DataFileSet, DataFolder
-from refiner.launchers import LaunchStats, LocalLauncher
-from refiner.media import VideoFile
+import refiner.text as text
+import refiner.video as video
 from refiner.pipeline import (
-    RefinerPipeline,
-    Row,
-    Shard,
     from_items,
     from_source,
     read_csv,
@@ -22,19 +20,11 @@ from refiner.worker.metrics.api import (
     log_throughput,
     register_gauge,
 )
-from refiner.worker.runner import Worker, WorkerRunStats
+
+robot = robotics
 
 __all__ = [
-    "RefinerPipeline",
-    "LocalLauncher",
-    "LaunchStats",
-    "DataFile",
-    "DataFolder",
-    "DataFileSet",
-    "Shard",
-    "Row",
-    "Worker",
-    "WorkerRunStats",
+    # sources
     "read_csv",
     "read_jsonl",
     "read_lerobot",
@@ -42,16 +32,22 @@ __all__ = [
     "from_items",
     "from_source",
     "task",
+    # metrics
     "log_throughput",
     "log_gauge",
     "log_gauges",
-    "register_gauge",
     "log_histogram",
+    "register_gauge",
+    # expressions
     "col",
     "lit",
     "coalesce",
     "if_else",
-    "VideoFile",
-    "Video",
+    # submodules
+    "io",
+    "pipeline",
+    "video",
+    "robot",
     "robotics",
+    "text",
 ]

macrodata_refiner-0.2.2/src/refiner/execution/asyncio/__init__.py

@@ -0,0 +1 @@
+

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2}/src/refiner/execution/operators/vectorized.py

@@ -10,7 +10,7 @@ from refiner.execution.tracking.shards import (
     count_table_by_shard,
     counts_delta,
 )
-from refiner.pipeline.data.tabular import repeat_scalar
+from refiner.pipeline.data.tabular import filter_table, repeat_scalar
 from refiner.pipeline.expressions import eval_expr_arrow
 from refiner.pipeline.steps import (
     CastStep,
@@ -69,14 +69,7 @@ def apply_vectorized_op(
         return out, None
 
     if isinstance(op, FilterExprStep):
-        mask = eval_expr_arrow(op.predicate, table)
-        next_table = (
-            table
-            if isinstance(mask, pa.Scalar) and bool(mask.as_py())
-            else (
-                table.slice(0, 0) if isinstance(mask, pa.Scalar) else table.filter(mask)
-            )
-        )
+        next_table = filter_table(table, op.predicate)
         next_shard_counts = count_table_by_shard(next_table)
         for shard_id in set(shard_counts) | set(next_shard_counts):
             previous = int(shard_counts.get(shard_id, 0))

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2}/src/refiner/io/datafolder.py

@@ -1,4 +1,4 @@
-from collections.abc import Iterable, Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from os import PathLike
 from typing import IO, Any, TypeAlias, Union, cast
 
@@ -113,6 +113,34 @@ class DataFolder(DirFileSystem):
             return self.abs_path(paths)
         return [self.abs_path(p) for p in paths]
 
+    def find(self, path: str, *args, **kwargs):
+        # Avoid DirFileSystem.find(): some backends (notably HF buckets) can leak
+        # sibling prefix matches like `root-2/...` or return the bare root entry,
+        # and DirFileSystem._relpath() asserts before we can filter them out.
+        """List paths under this folder, skipping backend results outside the base path."""
+        detail = kwargs.get("detail", False)
+        target = self._join(path.rstrip("/"))
+        ret = self.fs.find(target, *args, **kwargs)
+        target = target.rstrip("/")
+        target_prefix = target + self.fs.sep
+        alt_target = target[1:] if target.startswith(self.fs.sep) else None
+        alt_prefix = alt_target + self.fs.sep if alt_target is not None else None
+
+        def rel(p: str) -> str | None:
+            if p == target or (alt_target is not None and p == alt_target):
+                return path.rstrip("/")
+            if p.startswith(target_prefix):
+                suffix = p[len(target_prefix) :]
+            elif alt_prefix is not None and p.startswith(alt_prefix):
+                suffix = p[len(alt_prefix) :]
+            else:
+                return None
+            return suffix if path in {"", "/"} else f"{path.rstrip('/')}/{suffix}"
+
+        if detail:
+            return {r: info for p, info in ret.items() if (r := rel(p)) is not None}
+        return [r for p in ret if (r := rel(p)) is not None]
+
     def open_files(
         self, paths: Iterable[str], mode: str = "rb", **kwargs
     ) -> list[IO[Any]]:
@@ -159,3 +187,29 @@ class DataFolder(DirFileSystem):
 
     def files(self, relpaths: Iterable[str]) -> list[DataFile]:
         return [self.file(p) for p in relpaths]
+
+    def iter_files_with_sizes(
+        self, *, recursive: bool = False, **kwargs: Any
+    ) -> Iterator[tuple[DataFile, int | None]]:
+        if recursive:
+            found = self.find("", detail=True, **kwargs)
+            items: Iterable[tuple[str, Mapping[str, Any]]] = found.items()
+        else:
+            items = (
+                (str(info["name"]), info)
+                for info in self.ls("", detail=True, **kwargs)
+                if isinstance(info, Mapping)
+            )
+
+        for relpath, info in sorted(items, key=lambda item: item[0]):
+            info_dict = dict(info)
+            if info_dict.get("type") != "file":
+                continue
+            size = info_dict.get("size")
+            yield self.file(relpath), size if isinstance(size, int) else None
+
+    def iter_files(
+        self, *, recursive: bool = False, **kwargs: Any
+    ) -> Iterator[DataFile]:
+        for file, _ in self.iter_files_with_sizes(recursive=recursive, **kwargs):
+            yield file

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2}/src/refiner/io/fileset.py

@@ -170,62 +170,77 @@ class DataFileSet:
         exts = tuple(e.lower() for e in self.extensions)
         seen: set[tuple[int, str]] = set()
         expanded: list[tuple[DataFile, ...]] = []
-
-        def _append_file(out: list[DataFile], file: DataFile) -> None:
-            if exts and not file.path.lower().endswith(exts):
+        sizes = dict(self._sizes)
+
+        def _append_file(
+            out: list[DataFile],
+            file: DataFile,
+            *,
+            size: int | None = None,
+            apply_extensions: bool = True,
+        ) -> None:
+            if apply_extensions and exts and not file.path.lower().endswith(exts):
                 return
             key = (id(file.fs), file.path)
             if key in seen:
                 return
             seen.add(key)
             out.append(file)
+            if size is not None:
+                sizes[(len(expanded), file.abs_path())] = int(size)
 
         for entry in self.entries:
             files: list[DataFile] = []
+            if isinstance(entry, _PathSource) and not glob.has_magic(entry.path):
+                try:
+                    info = entry.fs.info(entry.path)
+                except FileNotFoundError:
+                    raise FileNotFoundError(
+                        f"Could not resolve input: {entry.fs.unstrip_protocol(entry.path)!r}"
+                    )
+                item_type = info.get("type")
+                if item_type == "directory":
+                    entry = DataFolder(path=entry.path, fs=entry.fs)
+                elif item_type == "file":
+                    entry = DataFile(fs=entry.fs, path=entry.path)
+                else:
+                    raise TypeError(
+                        f"Unsupported file type {item_type!r} for input: "
+                        f"{entry.fs.unstrip_protocol(entry.path)!r}"
+                    )
+
             if isinstance(entry, DataFile):
-                _append_file(files, entry)
+                _append_file(files, entry, apply_extensions=False)
             elif isinstance(entry, DataFolder):
-                paths = (
-                    sorted(entry.find(""))
-                    if self.recursive
-                    else sorted(
-                        e["name"] if isinstance(e, dict) else e
-                        for e in entry.ls("", detail=True)
-                        if not isinstance(e, dict) or e.get("type") == "file"
-                    )
-                )
-                for path in paths:
-                    _append_file(files, entry.file(path))
+                for file, size in entry.iter_files_with_sizes(recursive=self.recursive):
+                    _append_file(files, file, size=size)
             else:
                 next_fs, path = entry.fs, entry.path
                 if glob.has_magic(path):
-                    for expanded_path in sorted(next_fs.glob(path)):
-                        _append_file(files, DataFile(fs=next_fs, path=expanded_path))
-                elif next_fs.exists(path):
-                    if next_fs.isdir(path):
-                        paths = (
-                            sorted(next_fs.find(path))
-                            if self.recursive
-                            else sorted(
-                                e["name"] if isinstance(e, dict) else e
-                                for e in next_fs.ls(path, detail=True)
-                                if not isinstance(e, dict) or e.get("type") == "file"
-                            )
+                    matched = next_fs.glob(path, detail=True)
+                    items = matched.items()
+                    for expanded_path, info in sorted(items):
+                        if not isinstance(expanded_path, str) or not isinstance(
+                            info, Mapping
+                        ):
+                            continue
+                        if info.get("type") != "file":
+                            continue
+                        size = info.get("size")
+                        _append_file(
+                            files,
+                            DataFile(fs=next_fs, path=expanded_path),
+                            size=size if isinstance(size, int) else None,
                         )
-                        for expanded_path in paths:
-                            _append_file(
-                                files, DataFile(fs=next_fs, path=expanded_path)
-                            )
-                    else:
-                        _append_file(files, DataFile(fs=next_fs, path=path))
                 else:
-                    raise FileNotFoundError(
-                        f"Could not resolve input: {next_fs.unstrip_protocol(path)!r}"
+                    raise AssertionError(
+                        "non-glob _PathSource should have been resolved"
                     )
             expanded.append(tuple(files))
 
         out = tuple(expanded)
         object.__setattr__(self, "_expanded_sources", out)
+        object.__setattr__(self, "_sizes", sizes)
         return out
 
     def resolve_file(self, source_index: int, path: str) -> DataFile:

{macrodata_refiner-0.2.1 → macrodata_refiner-0.2.2}/src/refiner/launchers/base.py

@@ -1,20 +1,22 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from dataclasses import replace
 from typing import TYPE_CHECKING
-from uuid import uuid4
 import re
 import time
+from uuid import uuid4
 
 from loguru import logger
 
 from refiner.platform.auth import CredentialsError
 from refiner.platform.client.api import MacrodataClient
-from refiner.platform.client.http import sanitize_terminal_text
+from refiner.platform.client.http import request_json, sanitize_terminal_text
 from refiner.platform.manifest import build_run_manifest
 from refiner.worker.context import RunHandle
 from refiner.pipeline.planning import (
     PlannedStage,
+    StageComputeRequirements,
     compile_planned_stages,
     plan_pipeline_stages,
 )
@@ -34,6 +36,7 @@ class BaseLauncher(ABC):
         num_workers: int | None = None,
         heartbeat_interval_seconds: int | None = None,
         cpus_per_worker: int | None = None,
+        gpus_per_worker: int | None = None,
     ):
         if not name.strip():
             raise ValueError("name must be non-empty")
@@ -41,6 +44,7 @@ class BaseLauncher(ABC):
         self.name = name
         self.job_id = job_id or self._build_local_job_id(name)
         self.cpus_per_worker: int | None = None
+        self.gpus_per_worker: int | None = None
         if num_workers is not None:
             if num_workers <= 0:
                 raise ValueError("num_workers must be > 0")
@@ -53,6 +57,10 @@ class BaseLauncher(ABC):
             if cpus_per_worker <= 0:
                 raise ValueError("cpus_per_worker must be > 0")
             self.cpus_per_worker = cpus_per_worker
+        if gpus_per_worker is not None:
+            if gpus_per_worker <= 0:
+                raise ValueError("gpus_per_worker must be > 0")
+            self.gpus_per_worker = gpus_per_worker
 
     @staticmethod
     def _build_local_job_id(name: str) -> str:
@@ -81,6 +89,14 @@ class BaseLauncher(ABC):
         try:
             return MacrodataClient()
         except CredentialsError:
+            try:
+                request_json(
+                    method="GET",
+                    path="/api/me",
+                    timeout_s=2.0,
+                )
+            except Exception:
+                pass
             self._warn(
                 "platform integration disabled: no API key found in "
                 "MACRODATA_API_KEY or local credentials. "
@@ -113,8 +129,22 @@ class BaseLauncher(ABC):
         *,
         secret_values: tuple[str, ...] = (),
     ) -> dict[str, object]:
+        resolved_stages = [
+            replace(stage, compute=self._stage_compute_requirements(stage.compute))
+            for stage in (stages or self._planned_stages())
+        ]
         return compile_planned_stages(
-            stages or self._planned_stages(), secret_values=secret_values
+            resolved_stages,
+            secret_values=secret_values,
+        )
+
+    def _stage_compute_requirements(
+        self, compute: StageComputeRequirements
+    ) -> StageComputeRequirements:
+        return replace(
+            compute,
+            cpus_per_worker=self.cpus_per_worker,
+            gpus_per_worker=self.gpus_per_worker,
         )
 
     def _run_manifest(