flyte 2.0.0b17__py3-none-any.whl → 2.0.0b18__py3-none-any.whl

flyte/io/_hashing_io.py

@@ -0,0 +1,342 @@
+from __future__ import annotations
+
+import hashlib
+import inspect
+from typing import Any, Iterable, Optional, Protocol, Union, runtime_checkable
+
+
+@runtime_checkable
+class HashMethod(Protocol):
+    def update(self, data: memoryview, /) -> None: ...
+    def result(self) -> str: ...
+
+    # Optional convenience; not required by the writers.
+    def reset(self) -> None: ...
+
+
+class PrecomputedValue(HashMethod):
+    def __init__(self, value: str):
+        self._value = value
+
+    def update(self, data: memoryview, /) -> None: ...
+
+    def result(self) -> str:
+        return self._value
+
+
+class HashlibAccumulator(HashMethod):
+    """
+    Wrap a hashlib-like object to the Accumulator protocol.
+    h = hashlib.new("sha256")
+    acc = HashlibAccumulator(h)
+    """
+
+    def __init__(self, h):
+        self._h = h
+
+    def update(self, data: memoryview, /) -> None:
+        self._h.update(data)
+
+    def result(self) -> Any:
+        return self._h.hexdigest()
+
+    @classmethod
+    def from_hash_name(cls, name: str) -> HashlibAccumulator:
+        """
+        Create an accumulator from a hashlib algorithm name.
+        """
+        h = hashlib.new(name)
+        return cls(h)
+
+
+class HashingWriter:
+    """
+    Sync writer that updates a user-supplied accumulator on every write.
+
+    Hashing covers the exact bytes you pass in. If you write str, it is encoded
+    using the underlying file's .encoding if available, else UTF-8 (for hashing only).
+    """
+
+    def __init__(
+        self,
+        fh,
+        accumulator: HashMethod,
+        *,
+        encoding: Optional[str] = None,
+        errors: str = "strict",
+    ):
+        self._fh = fh
+        self._acc: HashMethod = accumulator
+        self._encoding = encoding or getattr(fh, "encoding", None)
+        self._errors = errors
+
+    def result(self) -> Any:
+        return self._acc.result()
+
+    def _to_bytes_mv(self, data) -> memoryview:
+        if isinstance(data, str):
+            b = data.encode(self._encoding or "utf-8", self._errors)
+            return memoryview(b)
+        if isinstance(data, (bytes, bytearray, memoryview)):
+            return memoryview(data)
+        # Accept any buffer-protocol object (e.g., numpy arrays)
+        return memoryview(data)
+
+    def write(self, data):
+        mv = self._to_bytes_mv(data)
+        self._acc.update(mv)
+        return self._fh.write(data)
+
+    def writelines(self, lines: Iterable[Union[str, bytes, bytearray, memoryview]]):
+        for line in lines:
+            self.write(line)
+
+    def flush(self):
+        return self._fh.flush()
+
+    def close(self):
+        return self._fh.close()
+
+    def __getattr__(self, name):
+        return getattr(self._fh, name)
+
+
+class AsyncHashingWriter:
+    """
+    Async version of HashingWriter with the same behavior.
+    """
+
+    def __init__(
+        self,
+        fh,
+        accumulator: HashMethod,
+        *,
+        encoding: Optional[str] = None,
+        errors: str = "strict",
+    ):
+        self._fh = fh
+        self._acc = accumulator
+        self._encoding = encoding or getattr(fh, "encoding", None)
+        self._errors = errors
+
+    def result(self) -> Any:
+        return self._acc.result()
+
+    def _to_bytes_mv(self, data) -> memoryview:
+        if isinstance(data, str):
+            b = data.encode(self._encoding or "utf-8", self._errors)
+            return memoryview(b)
+        if isinstance(data, (bytes, bytearray, memoryview)):
+            return memoryview(data)
+        return memoryview(data)
+
+    async def write(self, data):
+        mv = self._to_bytes_mv(data)
+        self._acc.update(mv)
+        return await self._fh.write(data)
+
+    async def writelines(self, lines: Iterable[Union[str, bytes, bytearray, memoryview]]):
+        for line in lines:
+            await self.write(line)
+
+    async def flush(self):
+        fn = getattr(self._fh, "flush", None)
+        if fn is None:
+            return None
+        res = fn()
+        if inspect.isawaitable(res):
+            return await res
+        return res
+
+    async def close(self):
+        fn = getattr(self._fh, "close", None)
+        if fn is None:
+            return None
+        res = fn()
+        if inspect.isawaitable(res):
+            return await res
+        return res
+
+    def __getattr__(self, name):
+        return getattr(self._fh, name)
+
+
+class HashingReader:
+    """
+    Sync reader that updates a user-supplied accumulator on every read operation.
+
+    If the underlying handle returns str (text mode), we encode it for hashing only,
+    using the handle's .encoding if present, else the explicit 'encoding' arg, else UTF-8.
+    """
+
+    def __init__(
+        self,
+        fh,
+        accumulator: HashMethod,
+        *,
+        encoding: Optional[str] = None,
+        errors: str = "strict",
+    ):
+        self._fh = fh
+        self._acc = accumulator
+        self._encoding = encoding or getattr(fh, "encoding", None)
+        self._errors = errors
+
+    def result(self) -> str:
+        return self._acc.result()
+
+    def _to_bytes_mv(self, data) -> Optional[memoryview]:
+        if data is None:
+            return None
+        if isinstance(data, str):
+            return memoryview(data.encode(self._encoding or "utf-8", self._errors))
+        if isinstance(data, (bytes, bytearray, memoryview)):
+            return memoryview(data)
+        # Accept any buffer-protocol object (rare for read paths, but safe)
+        return memoryview(data)
+
+    def read(self, size: int = -1):
+        data = self._fh.read(size)
+        mv = self._to_bytes_mv(data)
+        if mv is not None and len(mv) > 0:
+            self._acc.update(mv)
+        return data
+
+    def readline(self, size: int = -1):
+        line = self._fh.readline(size)
+        mv = self._to_bytes_mv(line)
+        if mv is not None and len(mv) > 0:
+            self._acc.update(mv)
+        return line
+
+    def readlines(self, hint: int = -1):
+        lines = self._fh.readlines(hint)
+        # Update in order to reflect exact concatenation
+        for line in lines:
+            mv = self._to_bytes_mv(line)
+            if mv is not None and len(mv) > 0:
+                self._acc.update(mv)
+        return lines
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        # Delegate to the underlying iterator to preserve semantics (including buffering),
+        # but intercept the produced line to hash it.
+        line = next(self._fh)
+        mv = self._to_bytes_mv(line)
+        if mv is not None and len(mv) > 0:
+            self._acc.update(mv)
+        return line
+
+    # ---- passthrough ----
+    def __getattr__(self, name):
+        # Avoid leaking private lookups to the underlying object if we're missing something internal
+        if name.startswith("_"):
+            raise AttributeError(name)
+        return getattr(self._fh, name)
+
+
+class AsyncHashingReader:
+    """
+    Async reader that updates a user-supplied accumulator on every read operation.
+
+    Works with aiofiles/fsspec async handles. `flush`/`close` may be awaitable or sync.
+    """
+
+    def __init__(
+        self,
+        fh,
+        accumulator: HashMethod,
+        *,
+        encoding: Optional[str] = None,
+        errors: str = "strict",
+    ):
+        self._fh = fh
+        self._acc = accumulator
+        self._encoding = encoding or getattr(fh, "encoding", None)
+        self._errors = errors
+
+    def result(self) -> str:
+        return self._acc.result()
+
+    def _to_bytes_mv(self, data) -> Optional[memoryview]:
+        if data is None:
+            return None
+        if isinstance(data, str):
+            return memoryview(data.encode(self._encoding or "utf-8", self._errors))
+        if isinstance(data, (bytes, bytearray, memoryview)):
+            return memoryview(data)
+        return memoryview(data)
+
+    async def read(self, size: int = -1):
+        data = await self._fh.read(size)
+        mv = self._to_bytes_mv(data)
+        if mv is not None and len(mv) > 0:
+            self._acc.update(mv)
+        return data
+
+    async def readline(self, size: int = -1):
+        line = await self._fh.readline(size)
+        mv = self._to_bytes_mv(line)
+        if mv is not None and len(mv) > 0:
+            self._acc.update(mv)
+        return line
+
+    async def readlines(self, hint: int = -1):
+        # Some async filehandles implement readlines(); if not, fall back to manual loop.
+        if hasattr(self._fh, "readlines"):
+            lines = await self._fh.readlines(hint)
+            for line in lines:
+                mv = self._to_bytes_mv(line)
+                if mv is not None and len(mv) > 0:
+                    self._acc.update(mv)
+            return lines
+        # Fallback: read all via iteration
+        lines = []
+        async for line in self:
+            lines.append(line)
+        return lines
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        # Prefer the underlying async iterator if present
+        anext_fn = getattr(self._fh, "__anext__", None)
+        if anext_fn is not None:
+            try:
+                line = await anext_fn()
+            except StopAsyncIteration:
+                raise
+            mv = self._to_bytes_mv(line)
+            if mv is not None and len(mv) > 0:
+                self._acc.update(mv)
+            return line
+
+        # Fallback to readline-based iteration
+        line = await self.readline()
+        if line == "" or line == b"":
+            raise StopAsyncIteration
+        return line
+
+    async def flush(self):
+        fn = getattr(self._fh, "flush", None)
+        if fn is None:
+            return None
+        res = fn()
+        return await res if inspect.isawaitable(res) else res
+
+    async def close(self):
+        fn = getattr(self._fh, "close", None)
+        if fn is None:
+            return None
+        res = fn()
+        return await res if inspect.isawaitable(res) else res
+
+    # ---- passthrough ----
+    def __getattr__(self, name):
+        if name.startswith("_"):
+            raise AttributeError(name)
+        return getattr(self._fh, name)

flyte/models.py

@@ -342,9 +342,15 @@ class NativeInterface:
         """
         # Convert positional arguments to keyword arguments
         if len(args) > len(self.inputs):
-            raise ValueError(f"Too many positional arguments provided, inputs {self.inputs.keys()}, args {len(args)}")
+            raise ValueError(
+                f"Too many positional arguments provided, expected inputs {self.inputs.keys()}, args {len(args)}"
+            )
         for arg, input_name in zip(args, self.inputs.keys()):
             kwargs[input_name] = arg
+        if len(kwargs) > len(self.inputs):
+            raise ValueError(
+                f"Too many keyword arguments provided, expected inputs {self.inputs.keys()}, args {kwargs.keys()}"
+            )
         return kwargs
 
     def get_input_types(self) -> Dict[str, Type]:
@@ -410,13 +416,15 @@ class SerializationContext:
     code_bundle: Optional[CodeBundle] = None
     input_path: str = "{{.input}}"
     output_path: str = "{{.outputPrefix}}"
-    _entrypoint_path: str = field(default="_bin/runtime.py", init=False)
+    interpreter_path: str = "/opt/venv/bin/python"
     image_cache: ImageCache | None = None
     root_dir: Optional[pathlib.Path] = None
 
-    def get_entrypoint_path(self, interpreter_path: str) -> str:
+    def get_entrypoint_path(self, interpreter_path: Optional[str] = None) -> str:
         """
         Get the entrypoint path for the task. This is used to determine the entrypoint for the task execution.
         :param interpreter_path: The path to the interpreter (python)
         """
-        return os.path.join(os.path.dirname(os.path.dirname(interpreter_path)), self._entrypoint_path)
+        if interpreter_path is None:
+            interpreter_path = self.interpreter_path
+        return os.path.join(os.path.dirname(interpreter_path), "runtime.py")

flyte/remote/_action.py

@@ -327,9 +327,10 @@ class Action(ToJSONMixin):
                     )
                 else:
                     details = await self.details()
+                    error_message = details.error_info.message if details.error_info else ""
                     console.print(
                         f"[bold red]Action '{self.name}' in Run '{self.run_name}'"
-                        f" exited unsuccessfully in state {self.phase} with error: {details.error_info}[/bold red]"
+                        f" exited unsuccessfully in state {self.phase} with error: {error_message}[/bold red]"
                     )
             return
 
@@ -371,9 +372,10 @@ class Action(ToJSONMixin):
                         if ad.pb2.status.phase == run_definition_pb2.PHASE_SUCCEEDED:
                             console.print(f"[bold green]Run '{self.run_name}' completed successfully.[/bold green]")
                         else:
+                            error_message = ad.error_info.message if ad.error_info else ""
                             console.print(
                                 f"[bold red]Run '{self.run_name}' exited unsuccessfully in state {ad.phase}"
-                                f" with error: {ad.error_info}[/bold red]"
+                                f" with error: {error_message}[/bold red]"
                             )
                         break
         except asyncio.CancelledError:

flyte/storage/_storage.py

@@ -14,6 +14,7 @@ from obstore.fsspec import register
 
 from flyte._initialize import get_storage
 from flyte._logging import logger
+from flyte.errors import InitializationError
 
 _OBSTORE_SUPPORTED_PROTOCOLS = ["s3", "gs", "abfs", "abfss"]
 
@@ -77,21 +78,36 @@ def get_configured_fsspec_kwargs(
     protocol: typing.Optional[str] = None, anonymous: bool = False
 ) -> typing.Dict[str, typing.Any]:
     if protocol:
+        # Try to get storage config safely - may not be initialized for local operations
+        try:
+            storage_config = get_storage()
+        except InitializationError:
+            storage_config = None
+
         match protocol:
             case "s3":
                 # If the protocol is s3, we can use the s3 filesystem
                 from flyte.storage import S3
 
+                if storage_config and isinstance(storage_config, S3):
+                    return storage_config.get_fsspec_kwargs(anonymous=anonymous)
+
                 return S3.auto().get_fsspec_kwargs(anonymous=anonymous)
             case "gs":
                 # If the protocol is gs, we can use the gs filesystem
                 from flyte.storage import GCS
 
+                if storage_config and isinstance(storage_config, GCS):
+                    return storage_config.get_fsspec_kwargs(anonymous=anonymous)
+
                 return GCS.auto().get_fsspec_kwargs(anonymous=anonymous)
             case "abfs" | "abfss":
                 # If the protocol is abfs or abfss, we can use the abfs filesystem
                 from flyte.storage import ABFS
 
+                if storage_config and isinstance(storage_config, ABFS):
+                    return storage_config.get_fsspec_kwargs(anonymous=anonymous)
+
                 return ABFS.auto().get_fsspec_kwargs(anonymous=anonymous)
             case _:
                 return {}
@@ -145,7 +161,6 @@ async def get(from_path: str, to_path: Optional[str | pathlib.Path] = None, recu
         else:
             exists = file_system.exists(from_path)
         if not exists:
-            # TODO: update exception to be more specific
             raise AssertionError(f"Unable to load data from {from_path}")
         file_system = _get_anonymous_filesystem(from_path)
         logger.debug(f"Attempting anonymous get with {file_system}")

flyte/types/_type_engine.py

@@ -39,7 +39,6 @@ from pydantic import BaseModel
 from typing_extensions import Annotated, get_args, get_origin
 
 import flyte.storage as storage
-from flyte._hash import HashMethod
 from flyte._logging import logger
 from flyte._utils.helpers import load_proto_from_file
 from flyte.models import NativeInterface
@@ -1096,23 +1095,6 @@ class TypeEngine(typing.Generic[T]):
         ):
             raise TypeTransformerFailedError(f"Python value cannot be None, expected {python_type}/{expected}")
 
-    @classmethod
-    def calculate_hash(cls, python_val: typing.Any, python_type: Type[T]) -> Optional[str]:
-        # In case the value is an annotated type we inspect the annotations and look for hash-related annotations.
-        hsh = None
-        if is_annotated(python_type):
-            # We are now dealing with one of two cases:
-            # 1. The annotated type is a `HashMethod`, which indicates that we should produce the hash using
-            #    the method indicated in the annotation.
-            # 2. The annotated type is being used for a different purpose other than calculating hash values,
-            #    in which case we should just continue.
-            for annotation in get_args(python_type)[1:]:
-                if not isinstance(annotation, HashMethod):
-                    continue
-                hsh = annotation.calculate(python_val)
-                break
-        return hsh
-
     @classmethod
     async def to_literal(
         cls, python_val: typing.Any, python_type: Type[T], expected: types_pb2.LiteralType
@@ -1125,8 +1107,6 @@ class TypeEngine(typing.Generic[T]):
         lv = await transformer.to_literal(python_val, python_type, expected)
 
         modify_literal_uris(lv)
-        calculated_hash = cls.calculate_hash(python_val, python_type) or ""
-        lv.hash = calculated_hash
         return lv
 
     @classmethod
@@ -1795,7 +1775,6 @@ class DictTransformer(TypeTransformer[dict]):
         for k, v in python_val.items():
             if type(k) is not str:
                 raise ValueError("Flyte MapType expects all keys to be strings")
-            # TODO: log a warning for Annotated objects that contain HashMethod
 
             _, v_type = self.extract_types(python_type)
             lit_map[k] = TypeEngine.to_literal(v, cast(type, v_type), expected.map_value_type)

{flyte-2.0.0b17.data → flyte-2.0.0b18.data}/scripts/runtime.py

@@ -101,6 +101,7 @@ def main(
     from flyte._logging import logger
     from flyte.models import ActionID, Checkpoints, CodeBundle, RawDataPath
 
+    logger.warning(f"Flyte runtime started for action {name} with run name {run_name}")
     logger.info("Registering faulthandler for SIGUSR1")
     faulthandler.register(signal.SIGUSR1)
 
@@ -168,6 +169,7 @@ def main(
         await controller.stop()
 
     asyncio.run(_run_and_stop())
+    logger.warning(f"Flyte runtime completed for action {name} with run name {run_name}")
 
 
 if __name__ == "__main__":

{flyte-2.0.0b17.dist-info → flyte-2.0.0b18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flyte
-Version: 2.0.0b17
+Version: 2.0.0b18
 Summary: Add your description here
 Author-email: Ketan Umare <kumare3@users.noreply.github.com>
 Requires-Python: >=3.10