datachain 0.31.3__tar.gz → 0.31.4__tar.gz

{datachain-0.31.3 → datachain-0.31.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.3
+Version: 0.31.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.3 → datachain-0.31.4}/src/datachain/catalog/catalog.py

@@ -144,26 +144,19 @@ def shutdown_process(
             return proc.wait()
 
 
-def process_output(stream: IO[bytes], callback: Callable[[str], None]) -> None:
+def _process_stream(stream: "IO[bytes]", callback: Callable[[str], None]) -> None:
     buffer = b""
+    while byt := stream.read(1):  # Read one byte at a time
+        buffer += byt
 
-    try:
-        while byt := stream.read(1):  # Read one byte at a time
-            buffer += byt
-
-            if byt in (b"\n", b"\r"):  # Check for newline or carriage return
-                line = buffer.decode("utf-8", errors="replace")
-                callback(line)
-                buffer = b""  # Clear buffer for the next line
-
-        if buffer:  # Handle any remaining data in the buffer
-            line = buffer.decode("utf-8", errors="replace")
+        if byt in (b"\n", b"\r"):  # Check for newline or carriage return
+            line = buffer.decode("utf-8")
             callback(line)
-    finally:
-        try:
-            stream.close()  # Ensure output is closed
-        except Exception:  # noqa: BLE001, S110
-            pass
+            buffer = b""  # Clear buffer for next line
+
+    if buffer:  # Handle any remaining data in the buffer
+        line = buffer.decode("utf-8")
+        callback(line)
 
 
 class DatasetRowsFetcher(NodesThreadPool):
@@ -1767,13 +1760,13 @@ class Catalog:
             recursive=recursive,
         )
 
-    @staticmethod
     def query(
+        self,
         query_script: str,
         env: Optional[Mapping[str, str]] = None,
         python_executable: str = sys.executable,
-        stdout_callback: Optional[Callable[[str], None]] = None,
-        stderr_callback: Optional[Callable[[str], None]] = None,
+        capture_output: bool = False,
+        output_hook: Callable[[str], None] = noop,
         params: Optional[dict[str, str]] = None,
         job_id: Optional[str] = None,
         interrupt_timeout: Optional[int] = None,
@@ -1788,18 +1781,13 @@ class Catalog:
             },
         )
         popen_kwargs: dict[str, Any] = {}
-
-        if stdout_callback is not None:
-            popen_kwargs = {"stdout": subprocess.PIPE}
-        if stderr_callback is not None:
-            popen_kwargs["stderr"] = subprocess.PIPE
+        if capture_output:
+            popen_kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.STDOUT}
 
         def raise_termination_signal(sig: int, _: Any) -> NoReturn:
             raise TerminationSignal(sig)
 
-        stdout_thread: Optional[Thread] = None
-        stderr_thread: Optional[Thread] = None
-
+        thread: Optional[Thread] = None
         with subprocess.Popen(cmd, env=env, **popen_kwargs) as proc:  # noqa: S603
             logger.info("Starting process %s", proc.pid)
 
@@ -1813,20 +1801,10 @@ class Catalog:
             orig_sigterm_handler = signal.getsignal(signal.SIGTERM)
             signal.signal(signal.SIGTERM, raise_termination_signal)
             try:
-                if stdout_callback is not None:
-                    stdout_thread = Thread(
-                        target=process_output,
-                        args=(proc.stdout, stdout_callback),
-                        daemon=True,
-                    )
-                    stdout_thread.start()
-                if stderr_callback is not None:
-                    stderr_thread = Thread(
-                        target=process_output,
-                        args=(proc.stderr, stderr_callback),
-                        daemon=True,
-                    )
-                    stderr_thread.start()
+                if capture_output:
+                    args = (proc.stdout, output_hook)
+                    thread = Thread(target=_process_stream, args=args, daemon=True)
+                    thread.start()
 
                 proc.wait()
             except TerminationSignal as exc:
@@ -1844,22 +1822,8 @@ class Catalog:
             finally:
                 signal.signal(signal.SIGTERM, orig_sigterm_handler)
                 signal.signal(signal.SIGINT, orig_sigint_handler)
-                # wait for the reader thread
-                thread_join_timeout_seconds = 30
-                if stdout_thread is not None:
-                    stdout_thread.join(timeout=thread_join_timeout_seconds)
-                    if stdout_thread.is_alive():
-                        logger.warning(
-                            "stdout thread is still alive after %s seconds",
-                            thread_join_timeout_seconds,
-                        )
-                if stderr_thread is not None:
-                    stderr_thread.join(timeout=thread_join_timeout_seconds)
-                    if stderr_thread.is_alive():
-                        logger.warning(
-                            "stderr thread is still alive after %s seconds",
-                            thread_join_timeout_seconds,
-                        )
+                if thread:
+                    thread.join()  # wait for the reader thread
 
         logger.info("Process %s exited with return code %s", proc.pid, proc.returncode)
         if proc.returncode in (

{datachain-0.31.3 → datachain-0.31.4}/src/datachain/lib/file.py

@@ -35,6 +35,7 @@ if TYPE_CHECKING:
     from datachain.catalog import Catalog
     from datachain.client.fsspec import Client
     from datachain.dataset import RowDict
+    from datachain.query.session import Session
 
 sha256 = partial(hashlib.sha256, usedforsecurity=False)
 
@@ -252,6 +253,15 @@ class File(DataModel):
         "last_modified",
     ]
 
+    # Allowed kwargs we forward to TextIOWrapper
+    _TEXT_WRAPPER_ALLOWED: ClassVar[tuple[str, ...]] = (
+        "encoding",
+        "errors",
+        "newline",
+        "line_buffering",
+        "write_through",
+    )
+
     @staticmethod
     def _validate_dict(
         v: Optional[Union[str, dict, list[dict]]],
@@ -328,7 +338,6 @@ class File(DataModel):
             from datachain.catalog.loader import get_catalog
 
             catalog = get_catalog()
-
         from datachain.client.fsspec import Client
 
         client_cls = Client.get_implementation(path)
@@ -341,6 +350,27 @@ class File(DataModel):
         file._set_stream(catalog)
         return file
 
+    @classmethod
+    def at(cls, uri: str, session: Optional["Session"] = None) -> "Self":
+        """Construct a File from a full URI in one call.
+
+        Example:
+            file = File.at("s3://bucket/path/to/output.png")
+            with file.open("wb") as f: ...
+        """
+        from datachain.client.fsspec import Client
+        from datachain.query.session import Session
+
+        if session is None:
+            session = Session.get()
+        catalog = session.catalog
+
+        client_cls = Client.get_implementation(uri)
+        source, rel_path = client_cls.split_url(uri)
+        file = cls(source=client_cls.get_uri(source), path=rel_path)
+        file._set_stream(catalog)
+        return file
+
     @classmethod
     def _from_row(cls, row: "RowDict") -> "Self":
         return cls(**{key: row[key] for key in cls._datachain_column_types})
@@ -354,28 +384,70 @@ class File(DataModel):
         return str(PurePosixPath(self.path).parent)
 
     @contextmanager
-    def open(self, mode: Literal["rb", "r"] = "rb") -> Iterator[Any]:
-        """Open the file and return a file object."""
-        if self.location:
-            with VFileRegistry.open(self, self.location) as f:  # type: ignore[arg-type]
-                yield f
+    def open(self, mode: str = "rb", **open_kwargs) -> Iterator[Any]:
+        """Open the file and return a file-like object.
 
-        else:
+        Supports both read ("rb", "r") and write modes (e.g. "wb", "w", "ab").
+        When opened in a write mode, metadata is refreshed after closing.
+        """
+        writing = any(ch in mode for ch in "wax+")
+        if self.location and writing:
+            raise VFileError(
+                "Writing to virtual file is not supported",
+                self.source,
+                self.path,
+            )
+
+        if self._catalog is None:
+            raise RuntimeError("Cannot open file: catalog is not set")
+
+        client: Client = self._catalog.get_client(self.source)
+
+        if not writing:
+            if self.location:
+                with VFileRegistry.open(self, self.location) as f:  # type: ignore[arg-type]
+                    yield self._wrap_text(f, mode, open_kwargs)
+                return
             if self._caching_enabled:
                 self.ensure_cached()
-            client: Client = self._catalog.get_client(self.source)
             with client.open_object(
                 self, use_cache=self._caching_enabled, cb=self._download_cb
             ) as f:
-                yield io.TextIOWrapper(f) if mode == "r" else f
+                yield self._wrap_text(f, mode, open_kwargs)
+            return
+
+        # write path
+        full_path = client.get_full_path(self.get_path_normalized())
+        with client.fs.open(full_path, mode, **open_kwargs) as f:
+            yield self._wrap_text(f, mode, open_kwargs)
+
+        # refresh metadata
+        info = client.fs.info(full_path)
+        refreshed = client.info_to_file(info, self.get_path_normalized())
+        for k, v in refreshed.model_dump().items():
+            setattr(self, k, v)
+
+    def _wrap_text(self, f: Any, mode: str, open_kwargs: dict[str, Any]) -> Any:
+        """Return stream possibly wrapped for text."""
+        if "b" in mode or isinstance(f, io.TextIOBase):
+            return f
+        filtered = {
+            k: open_kwargs[k] for k in self._TEXT_WRAPPER_ALLOWED if k in open_kwargs
+        }
+        return io.TextIOWrapper(f, **filtered)
 
     def read_bytes(self, length: int = -1):
         """Returns file contents as bytes."""
         with self.open() as stream:
             return stream.read(length)
 
-    def read_text(self):
-        """Returns file contents as text."""
+    def read_text(self, **open_kwargs):
+        """Return file contents decoded as text.
+
+        **open_kwargs : Any
+            Extra keyword arguments forwarded to ``open(mode="r", ...)``
+            (e.g. ``encoding="utf-8"``, ``errors="ignore"``)
+        """
         if self.location:
             raise VFileError(
                 "Reading text from virtual file is not supported",
@@ -383,7 +455,7 @@ class File(DataModel):
                 self.path,
             )
 
-        with self.open(mode="r") as stream:
+        with self.open(mode="r", **open_kwargs) as stream:
             return stream.read()
 
     def read(self, length: int = -1):
@@ -701,14 +773,19 @@ class TextFile(File):
     """`DataModel` for reading text files."""
 
     @contextmanager
-    def open(self, mode: Literal["rb", "r"] = "r"):
-        """Open the file and return a file object (default to text mode)."""
-        with super().open(mode=mode) as stream:
+    def open(self, mode: str = "r", **open_kwargs) -> Iterator[Any]:
+        """Open the file and return a file-like object.
+        Default to text mode"""
+        with super().open(mode=mode, **open_kwargs) as stream:
             yield stream
 
-    def read_text(self):
-        """Returns file contents as text."""
-        with self.open() as stream:
+    def read_text(self, **open_kwargs):
+        """Return file contents as text.
+
+        **open_kwargs : Any
+            Extra keyword arguments forwarded to ``open()`` (e.g. encoding).
+        """
+        with self.open(**open_kwargs) as stream:
             return stream.read()
 
     def save(self, destination: str, client_config: Optional[dict] = None):

{datachain-0.31.3 → datachain-0.31.4}/src/datachain.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datachain
-Version: 0.31.3
+Version: 0.31.4
 Summary: Wrangle unstructured AI data at scale
 Author-email: Dmitry Petrov <support@dvc.org>
 License-Expression: Apache-2.0

{datachain-0.31.3 → datachain-0.31.4}/tests/func/test_file.py

@@ -1,9 +1,12 @@
+import io
+
 import pytest
 import pytz
 
 import datachain as dc
 from datachain.data_storage.sqlite import SQLiteWarehouse
 from datachain.lib.file import File, FileError
+from datachain.query import C
 from datachain.utils import TIME_ZERO
 
 
@@ -91,3 +94,68 @@ def test_upload(cloud_test_catalog):
     assert f.read() == img_bytes
 
     client.fs.rm(dest, recursive=True)
+
+
+def test_open_write_binary(cloud_test_catalog):
+    ctc = cloud_test_catalog
+    catalog = ctc.catalog
+    src_uri = ctc.src_uri
+    data = b"hello via open()"
+    file_path = f"{src_uri}/test-open-write-bytes.bin"
+
+    file = File.at(file_path, ctc.session)
+    with file.open("wb") as f:
+        f.write(data)
+
+    assert file.size == len(data)
+    assert file.read() == data
+
+    # Query storage for exactly that relative path.
+    # Metadata already refreshed by open() write path.
+    rel_path = file.path
+    chain = dc.read_storage(src_uri, session=ctc.session).filter(
+        C("file.path") == rel_path
+    )
+    results = list(chain.to_values("file"))
+    assert len(results) == 1
+    match = results[0]
+    for field_name in File.model_fields:
+        if field_name == "last_modified":
+            # Allow up to 1s difference across backends
+            # (some backends don't keep microsecond precision, we keep it simple here)
+            assert match.last_modified.timestamp() == pytest.approx(
+                file.last_modified.timestamp(), abs=1
+            )
+        else:
+            assert getattr(match, field_name) == getattr(file, field_name), (
+                f"Mismatch in field '{field_name}'"
+            )
+
+    catalog.get_client(src_uri).fs.rm(file_path)
+
+
+def test_open_write_text(cloud_test_catalog):
+    ctc = cloud_test_catalog
+    catalog = ctc.catalog
+    src_uri = ctc.src_uri
+    file_path = f"{src_uri}/test-open-write-text.txt"
+    # Unicode content to exercise non-default (utf-16) encoding round trip
+    content = "Привет Мир\nSecond line"
+
+    file = File.at(file_path, ctc.session)
+    with file.open("w", encoding="utf-16-le") as f:
+        written_chars = f.write(content)
+
+    assert written_chars == len(content)
+    assert file.read_text(encoding="utf-16-le") == content
+
+    # Compute expected byte size using identical TextIOWrapper logic
+    buf = io.BytesIO()
+    tw = io.TextIOWrapper(buf, encoding="utf-16-le")
+    tw.write(content)
+    tw.flush()
+    expected_size = len(buf.getvalue())
+    tw.close()
+    assert file.size == expected_size
+
+    catalog.get_client(src_uri).fs.rm(file_path)

{datachain-0.31.3 → datachain-0.31.4}/tests/unit/test_query.py

@@ -42,31 +42,12 @@ def test_args(catalog, mock_popen):
     mock_popen.assert_called_once_with(["mypython", "-c", "pass"], env=expected_env)
 
 
-def test_capture_stdout(catalog, mock_popen):
-    mock_popen.stdout = io.BytesIO(b"Hello, World!\rLorem Ipsum\nDolor Sit Amet\nconse")
-    stdout = []
-
-    catalog.query("pass", stdout_callback=stdout.append)
-    assert stdout == ["Hello, World!\r", "Lorem Ipsum\n", "Dolor Sit Amet\n", "conse"]
-
-
-def test_capture_stderr(catalog, mock_popen):
-    mock_popen.stderr = io.BytesIO(b"Hello, World!\rLorem Ipsum\nDolor Sit Amet\nconse")
-    stderr = []
-
-    catalog.query("pass", stderr_callback=stderr.append)
-    assert stderr == ["Hello, World!\r", "Lorem Ipsum\n", "Dolor Sit Amet\n", "conse"]
-
-
 def test_capture_output(catalog, mock_popen):
     mock_popen.stdout = io.BytesIO(b"Hello, World!\rLorem Ipsum\nDolor Sit Amet\nconse")
-    mock_popen.stderr = io.BytesIO(b"foo\nbar")
-    stdout = []
-    stderr = []
+    lines = []
 
-    catalog.query("pass", stdout_callback=stdout.append, stderr_callback=stderr.append)
-    assert stdout == ["Hello, World!\r", "Lorem Ipsum\n", "Dolor Sit Amet\n", "conse"]
-    assert stderr == ["foo\n", "bar"]
+    catalog.query("pass", capture_output=True, output_hook=lines.append)
+    assert lines == ["Hello, World!\r", "Lorem Ipsum\n", "Dolor Sit Amet\n", "conse"]
 
 
 def test_canceled_by_user(catalog, mock_popen):