macrodata-refiner 0.2.0__tar.gz → 0.2.1__tar.gz

{macrodata_refiner-0.2.0/src/macrodata_refiner.egg-info → macrodata_refiner-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.2.0
+Version: 0.2.1
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "macrodata-refiner"
-version = "0.2.0"
+version = "0.2.1"
 description = "Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets"
 readme = "README.md"
 license = "Apache-2.0"

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1/src/macrodata_refiner.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: macrodata-refiner
-Version: 0.2.0
+Version: 0.2.1
 Summary: Refiner by Macrodata Labs, a data processing framework for Machine Learning large scale datasets
 Author: Macrodata Labs
 License-Expression: Apache-2.0

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/cli/auth.py

@@ -18,7 +18,7 @@ from refiner.platform.client import (
     sanitize_terminal_text,
     verify_api_key,
 )
-from refiner.cli.ui import display_identity, print_banner
+from refiner.cli.ui import display_identity, print_banner, stdin_is_interactive
 
 _TOKEN_SETTINGS_SUFFIX = "/settings/api-keys"
 
@@ -31,7 +31,7 @@ def _read_token(args: argparse.Namespace) -> str:
     if args.token and args.token.strip():
         return args.token.strip()
 
-    read_from_stdin = args.token_stdin or not sys.stdin.isatty()
+    read_from_stdin = args.token_stdin or not stdin_is_interactive()
     if read_from_stdin:
         token = sys.stdin.read().strip()
         if token:

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/cli/ui.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import sys
+
 from refiner.platform.client import UserIdentity
 
 ASCII_BANNER = r"""
@@ -26,3 +28,10 @@ def display_identity(user: UserIdentity) -> str:
     if email:
         return f"{label} ({email})"
     return label
+
+
+def stdin_is_interactive() -> bool:
+    try:
+        return sys.stdin.isatty()
+    except Exception:  # pragma: no cover
+        return False

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/execution/operators/row.py

@@ -58,15 +58,16 @@ def execute_row_steps(
             )
 
     async def _run_async_step(*, step: AsyncRowStep, row: Row) -> Row:
-        result = step.apply_row_async(row)
-        if inspect.isawaitable(result):
-            result = await result
-        result = cast(MapResult, result)
-        if isinstance(result, Row):
-            return result
-        if isinstance(result, dict):
-            return row.update(result)
-        raise TypeError(f"Unsupported map_async() result type: {type(result)!r}")
+        with set_active_step_index(step.index):
+            result = step.apply_row_async(row)
+            if inspect.isawaitable(result):
+                result = await result
+            result = cast(MapResult, result)
+            if isinstance(result, Row):
+                return result
+            if isinstance(result, dict):
+                return row.update(result)
+            raise TypeError(f"Unsupported map_async() result type: {type(result)!r}")
 
     def _run_step(i: int, *, flush_all: bool) -> None:
         step = ordered[i]

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/execution/operators/vectorized.py

@@ -22,6 +22,7 @@ from refiner.pipeline.steps import (
     VectorizedOp,
     WithColumnsStep,
 )
+from refiner.worker.context import set_active_step_index
 from refiner.worker.metrics.api import log_throughput
 
 
@@ -100,7 +101,8 @@ def apply_vectorized_op(
         return next_table, next_shard_counts
 
     if isinstance(op, FnTableStep):
-        next_table = op.fn(table)
+        with set_active_step_index(op.index):
+            next_table = op.fn(table)
         if not isinstance(next_table, pa.Table):
             raise TypeError(
                 f"map_table() must return pa.Table, got {type(next_table)!r}"

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/launchers/cloud.py

@@ -2,14 +2,16 @@ from __future__ import annotations
 
 import os
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 
+from refiner.cli.ui import stdin_is_interactive
 from refiner.platform.client import (
     CloudRunCreateRequest,
     CloudRuntimeConfig,
     StagePayload,
     serialize_pipeline_inline,
 )
+from refiner.platform.manifest import refiner_ref_exists_on_remote
 
 from refiner.launchers.base import BaseLauncher
 
@@ -17,6 +19,9 @@ if TYPE_CHECKING:
     from refiner.pipeline import RefinerPipeline
 
 
+_FALLBACK_ENV_VAR = "MACRODATA_FALLBACK_TO_LATEST_PYPI"
+
+
 @dataclass(frozen=True, slots=True)
 class CloudLaunchResult:
     job_id: str
@@ -96,6 +101,46 @@ class CloudLauncher(BaseLauncher):
                 )
         return {**(secrets or {}), **(env or {})} or None
 
+    @staticmethod
+    def _fallback_to_latest_pypi_enabled() -> bool:
+        raw = os.environ.get(_FALLBACK_ENV_VAR, "")
+        return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+    def _resolve_cloud_manifest(
+        self, *, secret_values: tuple[str, ...]
+    ) -> dict[str, object]:
+        manifest = self._run_manifest(secret_values=secret_values)
+        environment = manifest.get("environment")
+        if environment is None:
+            return manifest
+        environment_dict = cast(dict[str, object], environment)
+        refiner_ref = environment_dict.get("refiner_ref")
+        if not isinstance(refiner_ref, str) or not refiner_ref.strip():
+            return manifest
+        refiner_ref = refiner_ref.strip()
+        if refiner_ref_exists_on_remote(refiner_ref):
+            return manifest
+
+        message = (
+            f"Refiner ref {refiner_ref!r} is not available on GitHub. "
+            "Launch with the latest PyPI version instead?"
+        )
+        fallback_allowed = self._fallback_to_latest_pypi_enabled()
+        interactive = stdin_is_interactive()
+        if not fallback_allowed and interactive:
+            answer = input(f"{message} [y/N] ")
+            fallback_allowed = answer.strip().lower() in {"y", "yes"}
+        if fallback_allowed:
+            environment_dict["refiner_ref"] = None
+            return manifest
+        if interactive:
+            raise SystemExit("cloud launch aborted")
+
+        raise SystemExit(
+            f"{message} Launch aborted before submission. "
+            f"Set {_FALLBACK_ENV_VAR}=1 to allow fallback to the latest PyPI version."
+        )
+
     def launch(self) -> CloudLaunchResult:
         try:
             client = self._require_platform_client()
@@ -105,6 +150,7 @@ class CloudLauncher(BaseLauncher):
         resolved_env = self._resolve_env_values(self.env)
         secret_values = tuple(resolved_secrets.values()) if resolved_secrets else ()
         stages = self._planned_stages()
+        manifest = self._resolve_cloud_manifest(secret_values=secret_values)
         request = CloudRunCreateRequest(
             name=self.name,
             plan=self._compiled_plan(stages, secret_values=secret_values),
@@ -121,7 +167,7 @@ class CloudLauncher(BaseLauncher):
                 )
                 for stage in stages
             ],
-            manifest=self._run_manifest(secret_values=secret_values),
+            manifest=manifest,
             sync_local_dependencies=self.sync_local_dependencies,
             secrets=self._merged_env(resolved_secrets, resolved_env),
         )

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/pipeline/planning.py

@@ -313,6 +313,14 @@ def _step_payload(
     return payload
 
 
+def _sink_name_type(sink: Any) -> tuple[str, str, dict[str, Any] | None]:
+    payload = sink.describe()
+    if payload is not None:
+        return payload
+    sink_name = sink.__class__.__name__.replace("Sink", "").lower()
+    return sink_name or "sink", "writer", None
+
+
 def _serialize_args(
     args: dict[str, Any] | None, *, secret_values: tuple[str, ...] = ()
 ) -> dict[str, Any] | None:
@@ -389,16 +397,15 @@ def _compile_stage_steps(
             )
         )
 
-    sink_payload = pipeline.sink.describe() if pipeline.sink is not None else None
-    if sink_payload is not None:
-        base_name, step_type, args = sink_payload
+    if pipeline.sink is not None:
+        base_name, step_type, args = _sink_name_type(pipeline.sink)
         unique_name = _unique_name(base_name)
         steps.append(
             _step_payload(
                 name=unique_name,
                 step_type=step_type,
                 index=len(steps),
-                args=_serialize_args(args),
+                args=_serialize_args(args, secret_values=secret_values),
             )
         )
 

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/pipeline/sources/readers/lerobot.py

@@ -81,6 +81,10 @@ class LeRobotEpisodeReader(ParquetReader):
             split_row_groups=split_row_groups,
         )
 
+    def describe(self) -> dict[str, Any]:
+        inputs = [str(root.abs_paths("")) for root in self.roots]
+        return {"path": ", ".join(inputs), "inputs": inputs}
+
     def read_shard(self, shard: Shard) -> Iterator[SourceUnit]:
         """Read one planned episode shard and emit `LeRobotTabular` blocks."""
         descriptor = shard.descriptor

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/platform/manifest.py

@@ -6,8 +6,9 @@ import json
 import platform
 import subprocess
 import sys
+from urllib import error as urllib_error
+from urllib import request as urllib_request
 from collections.abc import Sequence
-from functools import lru_cache
 from importlib import metadata as importlib_metadata
 from pathlib import Path
 from typing import Any
@@ -90,36 +91,32 @@ def _collect_dependencies() -> list[dict[str, str]]:
 
 
 def _resolve_installed_version() -> str | None:
-    for package_name in ("refiner", "macrodata-refiner"):
-        try:
-            version = importlib_metadata.version(package_name).strip()
-        except importlib_metadata.PackageNotFoundError:
-            continue
-        if version:
-            return version
-    return None
+    try:
+        version = importlib_metadata.version("macrodata-refiner").strip()
+    except importlib_metadata.PackageNotFoundError:
+        return None
+    return version or None
 
 
 def _resolve_direct_url_git_sha() -> str | None:
-    for package_name in ("refiner", "macrodata-refiner"):
-        try:
-            dist = importlib_metadata.distribution(package_name)
-        except importlib_metadata.PackageNotFoundError:
-            continue
-        raw = dist.read_text("direct_url.json")
-        if not raw:
-            continue
-        try:
-            data = json.loads(raw)
-        except json.JSONDecodeError:
-            continue
-        vcs_info = data.get("vcs_info")
-        if not isinstance(vcs_info, dict):
-            continue
-        commit = str(vcs_info.get("commit_id", "")).strip()
-        if commit:
-            return commit
-    return None
+    try:
+        dist = importlib_metadata.distribution("macrodata-refiner")
+    except importlib_metadata.PackageNotFoundError:
+        return None
+    raw = dist.read_text("direct_url.json")
+    if not raw:
+        return None
+    try:
+        data = json.loads(raw)
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(data, dict):
+        return None
+    vcs_info = data.get("vcs_info")
+    if not isinstance(vcs_info, dict):
+        return None
+    commit = str(vcs_info.get("commit_id", "")).strip()
+    return commit or None
 
 
 def _resolve_repo_root(start: Path) -> Path | None:
@@ -148,19 +145,22 @@ def _resolve_local_repo_git_sha() -> str | None:
     return sha or None
 
 
-@lru_cache(maxsize=1)
-def _resolve_refiner_ref() -> str | None:
-    return (
-        _resolve_direct_url_git_sha()
-        or _resolve_local_repo_git_sha()
-        or _resolve_installed_version()
+def refiner_ref_exists_on_remote(ref: str) -> bool:
+    request = urllib_request.Request(
+        f"https://api.github.com/repos/macrodata-labs/refiner/commits/{ref}"
     )
+    try:
+        with urllib_request.urlopen(request):
+            return True
+    except (urllib_error.HTTPError, urllib_error.URLError):
+        return False
 
 
 def build_run_manifest(*, secret_values: Sequence[str] = ()) -> dict[str, Any]:
     script_path = _detect_script_path()
     path, text, sha256 = _read_script(script_path)
-    refiner_ref = _resolve_refiner_ref()
+    refiner_version = _resolve_installed_version()
+    refiner_ref = _resolve_direct_url_git_sha() or _resolve_local_repo_git_sha()
 
     manifest: dict[str, Any] = {
         "version": 1,
@@ -173,6 +173,7 @@ def build_run_manifest(*, secret_values: Sequence[str] = ()) -> dict[str, Any]:
         },
         "environment": {
             "python_version": platform.python_version(),
+            "refiner_version": refiner_version,
             "refiner_ref": refiner_ref,
             "platform": f"{platform.system().lower()}-{platform.machine().lower()}",
         },

{macrodata_refiner-0.2.0 → macrodata_refiner-0.2.1}/src/refiner/worker/runner.py

@@ -135,6 +135,9 @@ class Worker:
             runtime_name,
         )
         sink = self.pipeline.sink or NullSink()
+        sink_step_index = (
+            self.pipeline._next_step_index() if self.pipeline.sink is not None else None
+        )
 
         def _heartbeat_once() -> None:
             with inflight_lock:
@@ -166,7 +169,8 @@ class Worker:
                 if shard is None:
                     return
             try:
-                sink.on_shard_complete(shard_id)
+                with set_active_step_index(sink_step_index):
+                    sink.on_shard_complete(shard_id)
                 user_metrics_emitter.force_flush_user_metrics()
                 runtime_lifecycle.complete(shard)
             except Exception:  # noqa: BLE001
@@ -232,21 +236,26 @@ class Worker:
                     shard.start_key,
                     shard.end_key,
                 )
-                with set_active_step_index(0):
-                    obs_logger.info(
-                        "shard source started shard_id={} global_ordinal={}",
-                        shard.id,
-                        shard.global_ordinal,
-                    )
-                    for unit in self.pipeline.source.iter_shard_units(shard):
-                        rows = block_num_rows(unit)
-                        if rows > 0:
-                            rows_read += rows
-                            with inflight_lock:
-                                pending_rows_by_shard[shard.id] = (
-                                    pending_rows_by_shard.get(shard.id, 0) + rows
-                                )
-                        yield unit
+                obs_logger.info(
+                    "shard source started shard_id={} global_ordinal={}",
+                    shard.id,
+                    shard.global_ordinal,
+                )
+                source_iter = iter(self.pipeline.source.iter_shard_units(shard))
+                while True:
+                    with set_active_step_index(0):
+                        try:
+                            unit = next(source_iter)
+                        except StopIteration:
+                            break
+                    rows = block_num_rows(unit)
+                    if rows > 0:
+                        rows_read += rows
+                        with inflight_lock:
+                            pending_rows_by_shard[shard.id] = (
+                                pending_rows_by_shard.get(shard.id, 0) + rows
+                            )
+                    yield unit
                 obs_logger.info(
                     "shard source finished shard_id={} global_ordinal={} rows_read={}",
                     shard.id,
@@ -274,7 +283,8 @@ class Worker:
                     ):
                         if heartbeat_error is not None:
                             raise RuntimeError(f"heartbeat failed: {heartbeat_error}")
-                        written = sink.write_block(block)
+                        with set_active_step_index(sink_step_index):
+                            written = sink.write_block(block)
                         _apply_row_delta(
                             {
                                 shard_id: -count
@@ -334,7 +344,8 @@ class Worker:
                 stop_heartbeat.set()
                 heartbeat_thread.join(timeout=1.0)
                 try:
-                    sink.close()
+                    with set_active_step_index(sink_step_index):
+                        sink.close()
                 except Exception as e:
                     if execution_error is not None or run_exception is not None:
                         obs_logger.warning(