macrodata-refiner 0.3.0__tar.gz → 0.3.1__tar.gz

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.3.0
+Version: 0.3.1
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: aiohttp
 Requires-Dist: cloudpickle==3.1.2
 Requires-Dist: fsspec[http]
 Requires-Dist: httpx
@@ -29,10 +30,10 @@ Provides-Extra: hf
 Requires-Dist: datasets>=3.0.0; extra == "hf"
 Requires-Dist: huggingface-hub>=1.4.1; extra == "hf"
 Requires-Dist: hf>=1.7.1; extra == "hf"
-Provides-Extra: egocentric
-Requires-Dist: macrodata-refiner[hf]; extra == "egocentric"
-Requires-Dist: macrodata-refiner[video]; extra == "egocentric"
-Requires-Dist: ego-vision[models]>=0.1.8; extra == "egocentric"
+Provides-Extra: hand-tracking
+Requires-Dist: macrodata-refiner[hf]; extra == "hand-tracking"
+Requires-Dist: macrodata-refiner[video]; extra == "hand-tracking"
+Requires-Dist: ego-vision[models]>=0.1.25; extra == "hand-tracking"
 Provides-Extra: text
 Requires-Dist: warcio; extra == "text"
 Provides-Extra: hdf5
@@ -54,18 +55,18 @@ Provides-Extra: tfds
 Requires-Dist: macrodata-refiner[tensorflow]; extra == "tfds"
 Requires-Dist: tensorflow-datasets; extra == "tfds"
 Provides-Extra: testing
-Requires-Dist: macrodata-refiner[hdf5]; extra == "testing"
-Requires-Dist: macrodata-refiner[hf]; extra == "testing"
-Requires-Dist: macrodata-refiner[mcap]; extra == "testing"
-Requires-Dist: macrodata-refiner[video]; extra == "testing"
-Requires-Dist: macrodata-refiner[zarr]; extra == "testing"
-Requires-Dist: macrodata-refiner[text]; extra == "testing"
-Requires-Dist: macrodata-refiner[s3]; extra == "testing"
-Requires-Dist: macrodata-refiner[tfds]; extra == "testing"
+Requires-Dist: macrodata-refiner[all]; extra == "testing"
 Requires-Dist: pytest>=8.0.0; extra == "testing"
 Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
 Provides-Extra: all
-Requires-Dist: macrodata-refiner[testing]; extra == "all"
+Requires-Dist: macrodata-refiner[hdf5]; extra == "all"
+Requires-Dist: macrodata-refiner[hf]; extra == "all"
+Requires-Dist: macrodata-refiner[mcap]; extra == "all"
+Requires-Dist: macrodata-refiner[video]; extra == "all"
+Requires-Dist: macrodata-refiner[zarr]; extra == "all"
+Requires-Dist: macrodata-refiner[text]; extra == "all"
+Requires-Dist: macrodata-refiner[s3]; extra == "all"
+Requires-Dist: macrodata-refiner[tfds]; extra == "all"
 Dynamic: license-file
 
 <p align="center">

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "macrodata-refiner"
-version = "0.3.0"
+version = "0.3.1"
 description = "Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets"
 readme = "README.md"
 license = "Apache-2.0"
@@ -16,6 +16,7 @@ authors = [
 ]
 requires-python = ">=3.10"
 dependencies = [
+    "aiohttp",
     "cloudpickle==3.1.2",
     "fsspec[http]",
     "httpx",
@@ -38,10 +39,10 @@ hf = [
     "huggingface-hub>=1.4.1",
     "hf>=1.7.1",
 ]
-egocentric = [
+hand_tracking = [
     "macrodata-refiner[hf]",
     "macrodata-refiner[video]",
-    "ego-vision[models]>=0.1.8",
+    "ego-vision[models]>=0.1.25",
 ]
 text = [
     "warcio",
@@ -71,6 +72,11 @@ tfds = [
     "tensorflow-datasets",
 ]
 testing = [
+    "macrodata-refiner[all]",
+    "pytest>=8.0.0",
+    "pytest-cov>=5.0.0",
+]
+all = [
     "macrodata-refiner[hdf5]",
     "macrodata-refiner[hf]",
     "macrodata-refiner[mcap]",
@@ -79,11 +85,6 @@ testing = [
     "macrodata-refiner[text]",
     "macrodata-refiner[s3]",
     "macrodata-refiner[tfds]",
-    "pytest>=8.0.0",
-    "pytest-cov>=5.0.0",
-]
-all = [
-    "macrodata-refiner[testing]",
 ]
 
 [project.scripts]
@@ -102,7 +103,7 @@ refiner = ["py.typed"]
 
 [dependency-groups]
 dev = [
-    "macrodata-refiner[all]",
+    "macrodata-refiner[testing]",
     "pre-commit>=4.0.0",
     "ruff>=0.14.10",
     "ty>=0.0.7",

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/macrodata_refiner.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.3.0
+Version: 0.3.1
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: aiohttp
 Requires-Dist: cloudpickle==3.1.2
 Requires-Dist: fsspec[http]
 Requires-Dist: httpx
@@ -29,10 +30,10 @@ Provides-Extra: hf
 Requires-Dist: datasets>=3.0.0; extra == "hf"
 Requires-Dist: huggingface-hub>=1.4.1; extra == "hf"
 Requires-Dist: hf>=1.7.1; extra == "hf"
-Provides-Extra: egocentric
-Requires-Dist: macrodata-refiner[hf]; extra == "egocentric"
-Requires-Dist: macrodata-refiner[video]; extra == "egocentric"
-Requires-Dist: ego-vision[models]>=0.1.8; extra == "egocentric"
+Provides-Extra: hand-tracking
+Requires-Dist: macrodata-refiner[hf]; extra == "hand-tracking"
+Requires-Dist: macrodata-refiner[video]; extra == "hand-tracking"
+Requires-Dist: ego-vision[models]>=0.1.25; extra == "hand-tracking"
 Provides-Extra: text
 Requires-Dist: warcio; extra == "text"
 Provides-Extra: hdf5
@@ -54,18 +55,18 @@ Provides-Extra: tfds
 Requires-Dist: macrodata-refiner[tensorflow]; extra == "tfds"
 Requires-Dist: tensorflow-datasets; extra == "tfds"
 Provides-Extra: testing
-Requires-Dist: macrodata-refiner[hdf5]; extra == "testing"
-Requires-Dist: macrodata-refiner[hf]; extra == "testing"
-Requires-Dist: macrodata-refiner[mcap]; extra == "testing"
-Requires-Dist: macrodata-refiner[video]; extra == "testing"
-Requires-Dist: macrodata-refiner[zarr]; extra == "testing"
-Requires-Dist: macrodata-refiner[text]; extra == "testing"
-Requires-Dist: macrodata-refiner[s3]; extra == "testing"
-Requires-Dist: macrodata-refiner[tfds]; extra == "testing"
+Requires-Dist: macrodata-refiner[all]; extra == "testing"
 Requires-Dist: pytest>=8.0.0; extra == "testing"
 Requires-Dist: pytest-cov>=5.0.0; extra == "testing"
 Provides-Extra: all
-Requires-Dist: macrodata-refiner[testing]; extra == "all"
+Requires-Dist: macrodata-refiner[hdf5]; extra == "all"
+Requires-Dist: macrodata-refiner[hf]; extra == "all"
+Requires-Dist: macrodata-refiner[mcap]; extra == "all"
+Requires-Dist: macrodata-refiner[video]; extra == "all"
+Requires-Dist: macrodata-refiner[zarr]; extra == "all"
+Requires-Dist: macrodata-refiner[text]; extra == "all"
+Requires-Dist: macrodata-refiner[s3]; extra == "all"
+Requires-Dist: macrodata-refiner[tfds]; extra == "all"
 Dynamic: license-file
 
 <p align="center">

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/macrodata_refiner.egg-info/SOURCES.txt

@@ -131,7 +131,7 @@ src/refiner/platform/client/api.py
 src/refiner/platform/client/models.py
 src/refiner/platform/client/serialize.py
 src/refiner/robotics/__init__.py
-src/refiner/robotics/egocentric.py
+src/refiner/robotics/hand_tracking.py
 src/refiner/robotics/motion.py
 src/refiner/robotics/reward.py
 src/refiner/robotics/row.py

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/macrodata_refiner.egg-info/requires.txt

@@ -1,3 +1,4 @@
+aiohttp
 cloudpickle==3.1.2
 fsspec[http]
 httpx
@@ -10,12 +11,19 @@ msgspec>=0.20.0
 pydantic>=2.0.0
 
 [all]
-macrodata-refiner[testing]
+macrodata-refiner[hdf5]
+macrodata-refiner[hf]
+macrodata-refiner[mcap]
+macrodata-refiner[video]
+macrodata-refiner[zarr]
+macrodata-refiner[text]
+macrodata-refiner[s3]
+macrodata-refiner[tfds]
 
-[egocentric]
+[hand_tracking]
 macrodata-refiner[hf]
 macrodata-refiner[video]
-ego-vision[models]>=0.1.8
+ego-vision[models]>=0.1.25
 
 [hdf5]
 h5py
@@ -39,14 +47,7 @@ s3fs
 tensorflow
 
 [testing]
-macrodata-refiner[hdf5]
-macrodata-refiner[hf]
-macrodata-refiner[mcap]
-macrodata-refiner[video]
-macrodata-refiner[zarr]
-macrodata-refiner[text]
-macrodata-refiner[s3]
-macrodata-refiner[tfds]
+macrodata-refiner[all]
 pytest>=8.0.0
 pytest-cov>=5.0.0
 

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/refiner/execution/asyncio/window.py

@@ -2,7 +2,13 @@ from __future__ import annotations
 
 import heapq
 from collections.abc import Coroutine
-from concurrent.futures import ALL_COMPLETED, FIRST_COMPLETED, Future, wait
+from concurrent.futures import (
+    ALL_COMPLETED,
+    FIRST_COMPLETED,
+    FIRST_EXCEPTION,
+    Future,
+    wait,
+)
 from dataclasses import dataclass, field
 from typing import Generic, TypeVar
 
@@ -65,8 +71,8 @@ class AsyncWindow(Generic[T]):
         return self._take_ready()
 
     def drain(self) -> list[T]:
-        """Wait for all in-flight work, then return every available result."""
-        self._wait_until(return_when=ALL_COMPLETED)
+        """Wait for in-flight work, failing fast if any future raises."""
+        self._wait_until(return_when=FIRST_EXCEPTION)
         return self._take_ready()
 
     def cancel_pending(self) -> None:
@@ -83,13 +89,18 @@ class AsyncWindow(Generic[T]):
             return
         self._futures.difference_update(done)
         for future in done:
-            idx, value = future.result()
+            try:
+                idx, value = future.result()
+            except BaseException:
+                self.cancel_pending()
+                raise
             self._store_result(idx, value)
 
     def _wait_until(self, *, return_when: str) -> None:
         """Block until the requested completion condition and collect results."""
         while self._futures and (
-            return_when == ALL_COMPLETED or len(self._futures) >= self.max_in_flight
+            return_when in {ALL_COMPLETED, FIRST_EXCEPTION}
+            or len(self._futures) >= self.max_in_flight
         ):
             done, _ = wait(self._futures, return_when=return_when)
             self._collect_done(done)

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/refiner/execution/operators/row.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 import inspect
-from collections.abc import Iterable, Iterator, Sequence
+from collections.abc import Callable, Coroutine, Iterable, Iterator, Sequence
 from typing import cast
 
 from refiner.execution.asyncio.window import AsyncWindow
+from refiner.execution.asyncio.runtime import submit
 from refiner.execution.buffer import RowBuffer
 from refiner.execution.tracking.shards import ShardDeltaFn, ShardDeltaTracker
 from refiner.pipeline.data.row import Row
@@ -20,6 +21,8 @@ from refiner.pipeline.steps import (
 from refiner.worker.context import set_active_step_index
 from refiner.worker.metrics.api import register_gauge
 
+AsyncCloseFn = Callable[[], Coroutine[object, object, None]]
+
 
 def execute_row_steps(
     rows: Iterable[Row],
@@ -95,11 +98,16 @@ def execute_row_steps(
                 if window is None:
                     return
                 for row in inp.take_all():
-                    row.log_throughput("rows_processed", 1, unit="rows")
                     window.submit_blocking(_run_async_step(step=step, row=row))
-                out.extend(window.take_completed())
+                completed_rows = window.take_completed()
+                for row in completed_rows:
+                    row.log_throughput("rows_processed", 1, unit="rows")
+                out.extend(completed_rows)
                 if flush_all:
-                    out.extend(window.drain())
+                    drained_rows = window.drain()
+                    for row in drained_rows:
+                        row.log_throughput("rows_processed", 1, unit="rows")
+                    out.extend(drained_rows)
                 return
 
             if isinstance(step, FilterRowStep):
@@ -169,19 +177,37 @@ def execute_row_steps(
         for i in range(len(ordered)):
             _run_step(i, flush_all=flush_all)
 
+    def _close_async_steps() -> None:
+        for window in async_windows:
+            if window is not None:
+                window.cancel_pending()
+        close_fns: list[AsyncCloseFn] = []
+        for step in ordered:
+            if not isinstance(step, AsyncRowStep):
+                continue
+            fn = getattr(step, "fn", None)
+            close = getattr(fn, "aclose", None)
+            if close is not None:
+                close_fns.append(cast(AsyncCloseFn, close))
+        for close in close_fns:
+            submit(close()).result()
+
     def _drain_output() -> Iterator[Row]:
         outq = queues[-1]
         if not outq:
             return
         yield from outq.take_all()
 
-    for row in rows:
-        queues[0].append(row)
-        _pump(flush_all=False)
-        yield from _drain_output()
+    try:
+        for row in rows:
+            queues[0].append(row)
+            _pump(flush_all=False)
+            yield from _drain_output()
 
-    _pump(flush_all=True)
-    yield from _drain_output()
+        _pump(flush_all=True)
+        yield from _drain_output()
+    finally:
+        _close_async_steps()
 
 
 __all__ = ["execute_row_steps", "ShardDeltaFn"]

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/refiner/inference/__init__.py

@@ -38,6 +38,7 @@ from refiner.inference.types import (
     AnthropicFilePartProviderOptions,
     AnthropicProviderOptions,
     GoogleProviderOptions,
+    InferenceProvider,
     InferenceWarning,
     OpenAIProviderOptions,
 )
@@ -49,6 +50,7 @@ __all__ = [
     "GeneratePoolingMapFn",
     "GeneratePoolingPayload",
     "InferenceResponse",
+    "InferenceProvider",
     "InferenceAPICallError",
     "InferenceRetryError",
     "InferenceSchemaValidationError",

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/refiner/inference/capabilities.py

@@ -9,12 +9,16 @@ from refiner.inference.providers import (
     GoogleEndpointProvider,
     OpenAIEndpointProvider,
     OpenAIResponsesProvider,
-    VLLMProvider,
 )
 from refiner.inference.providers import anthropic as anthropic_provider
 from refiner.inference.providers import google as google_provider
 from refiner.inference.providers import openai as openai_provider
-from refiner.inference.types import InferenceWarning, Message, ModelCapabilities
+from refiner.inference.types import (
+    InferenceProvider,
+    InferenceWarning,
+    Message,
+    ModelCapabilities,
+)
 
 _MEDIA_WARNING_BYTES = 20 * 1024 * 1024
 _BASE64_CHARS = frozenset(
@@ -29,15 +33,7 @@ _TOOL_SETTINGS = {
 }
 
 
-def model_capabilities(
-    provider: (
-        AnthropicEndpointProvider
-        | GoogleEndpointProvider
-        | OpenAIEndpointProvider
-        | OpenAIResponsesProvider
-        | VLLMProvider
-    ),
-) -> ModelCapabilities:
+def model_capabilities(provider: InferenceProvider) -> ModelCapabilities:
     model = provider.model.lower()
     if isinstance(provider, GoogleEndpointProvider):
         return google_provider.model_capabilities(model)
@@ -52,13 +48,7 @@ def model_capabilities(
 
 def capability_warnings(
     *,
-    provider: (
-        AnthropicEndpointProvider
-        | GoogleEndpointProvider
-        | OpenAIEndpointProvider
-        | OpenAIResponsesProvider
-        | VLLMProvider
-    ),
+    provider: InferenceProvider,
     messages: Sequence[Message],
     params: Mapping[str, Any],
     provider_options: Mapping[str, Mapping[str, Any]] | None,
@@ -137,13 +127,7 @@ def capability_warnings(
 
 def _model_setting_warnings(
     *,
-    provider: (
-        AnthropicEndpointProvider
-        | GoogleEndpointProvider
-        | OpenAIEndpointProvider
-        | OpenAIResponsesProvider
-        | VLLMProvider
-    ),
+    provider: InferenceProvider,
     params: Mapping[str, Any],
     provider_options: Mapping[str, Mapping[str, Any]] | None,
 ) -> list[InferenceWarning]:

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/refiner/inference/generate_text.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import inspect
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import replace
-from typing import Any, Protocol, TypeAlias, cast
+from typing import Any, Protocol, cast
 
 from pydantic import BaseModel
 
@@ -33,18 +33,16 @@ from refiner.inference.providers.openai import (
     _OpenAIEndpointClient,
     _OpenAIResponsesClient,
 )
-from refiner.inference.types import InferenceWarning, Message, ProviderOptions
+from refiner.inference.types import (
+    InferenceProvider,
+    InferenceWarning,
+    Message,
+    ProviderOptions,
+)
 from refiner.pipeline.data.row import Row
+from refiner.pipeline.planning import describe_builtin
 from refiner.pipeline.steps import MapResult
 
-_InferenceProvider: TypeAlias = (
-    AnthropicEndpointProvider
-    | GoogleEndpointProvider
-    | OpenAIEndpointProvider
-    | OpenAIResponsesProvider
-    | VLLMProvider
-)
-
 
 class GenerateTextFn(Protocol):
     def __call__(
@@ -52,7 +50,7 @@ class GenerateTextFn(Protocol):
         *,
         messages: Sequence[Message] | None = None,
         raw_payload: Mapping[str, Any] | None = None,
-        providerOptions: ProviderOptions | None = None,
+        provider_options: ProviderOptions | None = None,
         maxRetries: int | None = None,
         schema: type[BaseModel] | None = None,
         schemaStrict: bool = True,
@@ -66,16 +64,36 @@ GenerateTextMapFn = Callable[[Row, GenerateTextFn], Awaitable[MapResult] | MapRe
 def generate_text(
     *,
     fn: GenerateTextMapFn,
-    provider: _InferenceProvider,
+    provider: InferenceProvider,
     default_generation_params: Mapping[str, Any] | None = None,
     max_concurrent_requests: int = 256,
 ) -> Callable[[Row], Awaitable[MapResult]]:
+    """Return an async row mapper that issues text-generation requests.
+
+    Args:
+        fn: Row-level function that receives the input row and a request function.
+            The request function accepts either typed messages or a raw provider
+            payload and returns an ``InferenceResponse``.
+        provider: Endpoint or runtime-service provider used to execute requests.
+        default_generation_params: Parameters merged into each typed message or
+            raw payload request unless overridden by that individual request.
+        max_concurrent_requests: Maximum number of provider requests allowed to
+            run at once per worker.
+    """
+
+    @describe_builtin(
+        "inference.generate_text",
+        fn=fn,
+        provider=provider.to_builtin_args(),
+        max_concurrent_requests=max_concurrent_requests,
+        default_generation_params=dict(default_generation_params or {}),
+    )
     async def _map(row: Row, request: RequestFn) -> MapResult:
         async def _generate_text(
             *,
             messages: Sequence[Message] | None = None,
             raw_payload: Mapping[str, Any] | None = None,
-            providerOptions: ProviderOptions | None = None,
+            provider_options: ProviderOptions | None = None,
             maxRetries: int | None = None,
             schema: type[BaseModel] | None = None,
             schemaStrict: bool = True,
@@ -83,7 +101,6 @@ def generate_text(
         ) -> InferenceResponse:
             if (messages is None) == (raw_payload is None):
                 raise ValueError("pass exactly one of messages or raw_payload")
-            provider_options = providerOptions
             max_retries = maxRetries
             schema_strict = schemaStrict
             schema_info = normalize_schema(
@@ -95,7 +112,7 @@ def generate_text(
             if raw_payload is not None:
                 if provider_options is not None:
                     raise ValueError(
-                        "providerOptions are not supported with raw_payload"
+                        "provider_options are not supported with raw_payload"
                     )
                 if schema is not None:
                     raise ValueError("schema is not supported with raw_payload")
@@ -174,7 +191,7 @@ async def _generate(
 
 def _build_payload(
     *,
-    provider: _InferenceProvider,
+    provider: InferenceProvider,
     messages: Sequence[Message],
     params: Mapping[str, Any],
     provider_options: ProviderOptions | None,
@@ -211,7 +228,7 @@ def _build_payload(
 
 
 def _provider_warnings(
-    provider: _InferenceProvider,
+    provider: InferenceProvider,
     provider_options: ProviderOptions | None,
 ) -> list[InferenceWarning]:
     if isinstance(provider, OpenAIResponsesProvider):

{macrodata_refiner-0.3.0 → macrodata_refiner-0.3.1}/src/refiner/inference/internal/message_conversion.py

@@ -20,7 +20,7 @@ def _custom_provider_data(
 
 
 def _provider_option(part: Mapping[str, Any], provider: str, key: str) -> object:
-    provider_options = part.get("providerOptions")
+    provider_options = part.get("provider_options")
     if not isinstance(provider_options, Mapping):
         return None
     options = cast(Mapping[str, Any], provider_options).get(provider)